2024-09-13T14:03:32.126 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T14:03:32.152 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T14:03:32.256 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557 branch: wip-aclamk-cbt-combined description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: akupczyk@redhat.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7903557' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-aclamk-cbt-combined ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-aclamk-cbt-combined sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 owner: scheduled_akupczyk@teuthology priority: 160 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 6545 sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 682/13280 suite: upgrade suite_branch: wip-aclamk-cbt-combined suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 targets: smithi042.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEc7Bp6hcVJggs1NaK0iFua6jFGsz3Gzg8NGspW4tJ8MrRoRmxUuyitZIco70DViXtjzGvgqDnOh8QokJTLrXeo= smithi159.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEuwuVyu0b1ZH092Pm1na/sVOhGOjXFm0YWEySgHzCjharMK1GiITrekxPdA/kNsWVumyWME3pnQRljYnePTHbE= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_10:00:18 tube: smithi user: akupczyk verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T14:03:32.257 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa; will attempt to use it 2024-09-13T14:03:32.257 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks 2024-09-13T14:03:32.257 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T14:03:32.259 INFO:teuthology.task.internal:Checking packages... 2024-09-13T14:03:32.281 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'd58f73069a8157a9ace12a0247ee3e8497707ea9' 2024-09-13T14:03:32.281 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:03:32.281 INFO:teuthology.packaging:ref: None 2024-09-13T14:03:32.281 INFO:teuthology.packaging:tag: None 2024-09-13T14:03:32.281 INFO:teuthology.packaging:branch: wip-aclamk-cbt-combined 2024-09-13T14:03:32.282 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:03:32.282 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-aclamk-cbt-combined 2024-09-13T14:03:32.432 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4697.gd58f7306 2024-09-13T14:03:32.433 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T14:03:32.440 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T14:03:32.440 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T14:03:32.528 INFO:teuthology.task.internal:Saving configuration 2024-09-13T14:03:32.552 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T14:03:32.578 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T14:03:32.606 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi042.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:58:27.145639', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEc7Bp6hcVJggs1NaK0iFua6jFGsz3Gzg8NGspW4tJ8MrRoRmxUuyitZIco70DViXtjzGvgqDnOh8QokJTLrXeo='} 2024-09-13T14:03:32.637 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi159.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 13:58:27.143877', 'locked_by': 'scheduled_akupczyk@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEuwuVyu0b1ZH092Pm1na/sVOhGOjXFm0YWEySgHzCjharMK1GiITrekxPdA/kNsWVumyWME3pnQRljYnePTHbE='} 2024-09-13T14:03:32.637 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T14:03:32.643 INFO:teuthology.task.internal:roles: ubuntu@smithi042.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T14:03:32.644 INFO:teuthology.task.internal:roles: ubuntu@smithi159.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T14:03:32.644 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T14:03:32.822 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5000d0fbe0>, signals=[15]) 2024-09-13T14:03:32.822 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T14:03:32.829 INFO:teuthology.task.internal:Opening connections... 2024-09-13T14:03:32.829 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T14:03:32.831 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:03:32.920 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi159.front.sepia.ceph.com 2024-09-13T14:03:32.920 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:03:33.025 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T14:03:33.033 DEBUG:teuthology.orchestra.run.smithi042:> uname -m 2024-09-13T14:03:33.051 INFO:teuthology.orchestra.run.smithi042.stdout:x86_64 2024-09-13T14:03:33.052 DEBUG:teuthology.orchestra.run.smithi042:> cat /etc/os-release 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:NAME="CentOS Stream" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION="9" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:ID="centos" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:ID_LIKE="rhel fedora" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION_ID="9" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:PLATFORM_ID="platform:el9" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:ANSI_COLOR="0;31" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:LOGO="fedora-logo-icon" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T14:03:33.110 INFO:teuthology.orchestra.run.smithi042.stdout:HOME_URL="https://centos.org/" 2024-09-13T14:03:33.111 INFO:teuthology.orchestra.run.smithi042.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T14:03:33.111 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T14:03:33.111 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T14:03:33.111 INFO:teuthology.lock.ops:Updating smithi042.front.sepia.ceph.com on lock server 2024-09-13T14:03:33.139 DEBUG:teuthology.orchestra.run.smithi159:> uname -m 2024-09-13T14:03:33.157 INFO:teuthology.orchestra.run.smithi159.stdout:x86_64 2024-09-13T14:03:33.158 DEBUG:teuthology.orchestra.run.smithi159:> cat /etc/os-release 2024-09-13T14:03:33.215 INFO:teuthology.orchestra.run.smithi159.stdout:NAME="CentOS Stream" 2024-09-13T14:03:33.215 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION="9" 2024-09-13T14:03:33.215 INFO:teuthology.orchestra.run.smithi159.stdout:ID="centos" 2024-09-13T14:03:33.215 INFO:teuthology.orchestra.run.smithi159.stdout:ID_LIKE="rhel fedora" 2024-09-13T14:03:33.215 INFO:teuthology.orchestra.run.smithi159.stdout:VERSION_ID="9" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:PLATFORM_ID="platform:el9" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:ANSI_COLOR="0;31" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:LOGO="fedora-logo-icon" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:HOME_URL="https://centos.org/" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T14:03:33.216 INFO:teuthology.orchestra.run.smithi159.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T14:03:33.216 INFO:teuthology.lock.ops:Updating smithi159.front.sepia.ceph.com on lock server 2024-09-13T14:03:33.239 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T14:03:33.255 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T14:03:33.359 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T14:03:33.360 DEBUG:teuthology.orchestra.run.smithi042:> test '!' -e /home/ubuntu/cephtest 2024-09-13T14:03:33.362 DEBUG:teuthology.orchestra.run.smithi159:> test '!' -e /home/ubuntu/cephtest 2024-09-13T14:03:33.379 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T14:03:33.678 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T14:03:33.679 DEBUG:teuthology.orchestra.run.smithi042:> test -z $(ls -A /var/lib/ceph) 2024-09-13T14:03:33.681 DEBUG:teuthology.orchestra.run.smithi159:> test -z $(ls -A /var/lib/ceph) 2024-09-13T14:03:33.708 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T14:03:33.795 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T14:03:33.821 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T14:03:33.821 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T14:03:33.821 DEBUG:teuthology.orchestra.run.smithi042:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T14:03:33.822 DEBUG:teuthology.orchestra.run.smithi159:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T14:03:33.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:03:33.839 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-09-13T14:03:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:03:33.840 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2024-09-13T14:03:33.897 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-503.el9.x86_64 2024-09-13T14:03:33.897 INFO:teuthology.task.kernel:Running kernel on smithi042: 5.14.0-503.el9.x86_64 2024-09-13T14:03:33.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum install -y kernel 2024-09-13T14:03:33.898 INFO:teuthology.orchestra.run.smithi159.stdout:5.14.0-503.el9.x86_64 2024-09-13T14:03:33.898 INFO:teuthology.task.kernel:Running kernel on smithi159: 5.14.0-503.el9.x86_64 2024-09-13T14:03:33.898 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum install -y kernel 2024-09-13T14:03:36.332 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - BaseOS 15 MB/s | 8.2 MB 00:00 2024-09-13T14:03:36.701 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - BaseOS 8.0 MB/s | 8.2 MB 00:01 2024-09-13T14:03:38.710 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-09-13T14:03:38.766 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - AppStream 16 MB/s | 20 MB 00:01 2024-09-13T14:03:44.153 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - CRB 4.7 MB/s | 6.5 MB 00:01 2024-09-13T14:03:45.066 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-13T14:03:45.762 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - Extras packages 68 kB/s | 19 kB 00:00 2024-09-13T14:03:46.484 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - Extras packages 83 kB/s | 19 kB 00:00 2024-09-13T14:03:48.016 INFO:teuthology.orchestra.run.smithi042.stdout:Extra Packages for Enterprise Linux 11 MB/s | 23 MB 00:02 2024-09-13T14:03:48.163 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 15 MB/s | 23 MB 00:01 2024-09-13T14:03:53.602 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 18 kB/s | 1.7 kB 00:00 2024-09-13T14:03:53.830 INFO:teuthology.orchestra.run.smithi042.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-13T14:03:55.625 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:03:55.625 INFO:teuthology.orchestra.run.smithi159.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:03:55.704 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:03:55.709 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:03:55.710 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout:Install 4 Packages 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 79 M 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 126 M 2024-09-13T14:03:55.711 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:03:55.937 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T14:03:55.937 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T14:03:56.036 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout: Package Architecture Version Repository Size 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:03:56.045 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout:Install 4 Packages 2024-09-13T14:03:56.046 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:03:56.047 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 79 M 2024-09-13T14:03:56.047 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 126 M 2024-09-13T14:03:56.047 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:03:56.403 INFO:teuthology.orchestra.run.smithi159.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 40 kB/s | 23 kB 00:00 2024-09-13T14:03:56.640 INFO:teuthology.orchestra.run.smithi042.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 42 kB/s | 23 kB 00:00 2024-09-13T14:03:57.237 INFO:teuthology.orchestra.run.smithi159.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-13T14:03:57.615 INFO:teuthology.orchestra.run.smithi042.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-09-13T14:03:57.895 INFO:teuthology.orchestra.run.smithi159.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-13T14:03:58.174 INFO:teuthology.orchestra.run.smithi042.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-13T14:03:58.604 INFO:teuthology.orchestra.run.smithi159.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 13 MB/s | 29 MB 00:02 2024-09-13T14:03:58.604 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:03:58.605 INFO:teuthology.orchestra.run.smithi159.stdout:Total 27 MB/s | 79 MB 00:02 2024-09-13T14:03:58.999 INFO:teuthology.orchestra.run.smithi042.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 12 MB/s | 29 MB 00:02 2024-09-13T14:03:58.999 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:03:59.000 INFO:teuthology.orchestra.run.smithi042.stdout:Total 27 MB/s | 79 MB 00:02 2024-09-13T14:03:59.278 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:03:59.647 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:03:59.647 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:03:59.709 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:04:00.107 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:04:00.107 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:04:00.594 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:04:00.594 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:04:01.120 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:04:01.120 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:04:03.061 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:04:03.702 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:04:03.778 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:04:03.871 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:04.359 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:04:04.451 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:06.700 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:06.842 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:07.226 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:07.355 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:11.641 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:11.760 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:12.515 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:12.556 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:16.611 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:17.698 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:54.778 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:54.818 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:55.148 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:55.149 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:04:55.149 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:55.149 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:56.014 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:56.014 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:04:56.014 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:04:56.014 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.014 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.015 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.015 INFO:teuthology.orchestra.run.smithi159.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.015 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:04:56.015 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:04:56.335 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:04:56.479 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.480 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:04:56.480 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:04:56.480 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:04:56.480 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:04:56.480 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T14:04:56.480 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi159.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:04:56.481 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:04:56.540 INFO:teuthology.orchestra.run.smithi159.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:56.540 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:04:56.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo rpm -qi grub2-tools 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Name : grub2-tools 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Epoch : 1 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Version : 2.06 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Release : 82.el9 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Architecture: x86_64 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:04:56.573 INFO:teuthology.orchestra.run.smithi159.stdout:Group : Unspecified 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Size : 8274670 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:License : GPLv3+ 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Packager : builder@centos.org 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Vendor : CentOS 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Summary : Support tools for GRUB. 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:Description : 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:04:56.574 INFO:teuthology.orchestra.run.smithi159.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:04:56.575 INFO:teuthology.orchestra.run.smithi159.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:04:56.575 INFO:teuthology.orchestra.run.smithi159.stdout:hardware devices. 2024-09-13T14:04:56.575 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:04:56.575 INFO:teuthology.orchestra.run.smithi159.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:04:56.576 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:04:56.576 INFO:teuthology.task.kernel:Updating grub on smithi159 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:04:56.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:04:56.612 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:56.653 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:57.028 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:57.028 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T14:04:57.029 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T14:04:57.029 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T14:04:57.227 INFO:teuthology.orchestra.run.smithi159.stderr:Generating grub configuration file ... 2024-09-13T14:04:57.800 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T14:04:57.800 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:04:57.800 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:04:57.800 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:57.800 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T14:04:57.801 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T14:04:57.801 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T14:04:57.801 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:04:57.801 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:04:58.118 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:04:58.204 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:58.204 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:04:58.204 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T14:04:58.204 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T14:04:58.205 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T14:04:58.205 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T14:04:58.205 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi042.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T14:04:58.205 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T14:04:58.263 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T14:04:58.263 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T14:04:58.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo rpm -qi grub2-tools 2024-09-13T14:04:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:Name : grub2-tools 2024-09-13T14:04:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:Epoch : 1 2024-09-13T14:04:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:Version : 2.06 2024-09-13T14:04:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:Release : 82.el9 2024-09-13T14:04:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:Architecture: x86_64 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Group : Unspecified 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Size : 8274670 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:License : GPLv3+ 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Packager : builder@centos.org 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:Vendor : CentOS 2024-09-13T14:04:58.304 INFO:teuthology.orchestra.run.smithi042.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:Summary : Support tools for GRUB. 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:Description : 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:hardware devices. 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:04:58.305 INFO:teuthology.orchestra.run.smithi042.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T14:04:58.306 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T14:04:58.306 INFO:teuthology.task.kernel:Updating grub on smithi042 to boot 5.14.0-505.el9.x86_64 2024-09-13T14:04:58.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T14:04:58.580 INFO:teuthology.orchestra.run.smithi159.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:04:58.609 INFO:teuthology.orchestra.run.smithi159.stderr:done 2024-09-13T14:04:58.611 DEBUG:teuthology.orchestra.run.smithi159:> mktemp 2024-09-13T14:04:58.626 INFO:teuthology.orchestra.run.smithi159.stdout:/tmp/tmp.E2vk7fvgcW 2024-09-13T14:04:58.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.E2vk7fvgcW 2024-09-13T14:04:58.693 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0666 /tmp/tmp.E2vk7fvgcW 2024-09-13T14:04:58.832 DEBUG:teuthology.orchestra.remote:smithi159:/tmp/tmp.E2vk7fvgcW is 6KB 2024-09-13T14:04:58.880 DEBUG:teuthology.orchestra.run.smithi159:> rm -fr /tmp/tmp.E2vk7fvgcW 2024-09-13T14:04:58.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:04:58.949 INFO:teuthology.orchestra.run.smithi042.stderr:Generating grub configuration file ... 2024-09-13T14:04:58.961 INFO:teuthology.orchestra.run.smithi159.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:04:58.961 INFO:teuthology.orchestra.run.smithi159.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:04:58.961 INFO:teuthology.orchestra.run.smithi159.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:04:58.962 INFO:teuthology.orchestra.run.smithi159.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:04:58.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:04:59.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo shutdown -r now 2024-09-13T14:05:00.339 INFO:teuthology.orchestra.run.smithi042.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T14:05:00.369 INFO:teuthology.orchestra.run.smithi042.stderr:done 2024-09-13T14:05:00.374 DEBUG:teuthology.orchestra.run.smithi042:> mktemp 2024-09-13T14:05:00.389 INFO:teuthology.orchestra.run.smithi042.stdout:/tmp/tmp.jYqxsbHSox 2024-09-13T14:05:00.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.jYqxsbHSox 2024-09-13T14:05:00.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0666 /tmp/tmp.jYqxsbHSox 2024-09-13T14:05:00.606 DEBUG:teuthology.orchestra.remote:smithi042:/tmp/tmp.jYqxsbHSox is 6KB 2024-09-13T14:05:00.654 DEBUG:teuthology.orchestra.run.smithi042:> rm -fr /tmp/tmp.jYqxsbHSox 2024-09-13T14:05:00.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T14:05:00.735 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T14:05:00.736 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T14:05:00.736 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T14:05:00.736 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T14:05:00.737 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T14:05:00.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo shutdown -r now 2024-09-13T14:05:29.149 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T14:05:29.150 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:05:29.152 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:05:30.886 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T14:05:30.886 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:05:30.887 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:05:47.709 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2024-09-13T14:05:49.245 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.42 2024-09-13T14:05:56.717 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:05:56.718 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:05:58.247 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:05:58.248 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:05:59.805 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2024-09-13T14:06:01.309 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.42 2024-09-13T14:06:11.811 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:06:11.812 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:06:13.311 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:06:13.312 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:06:30.205 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.159 2024-09-13T14:06:45.145 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-09-13T14:06:45.206 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:06:45.207 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:06:45.495 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:06:45.495 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:06:45.495 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-09-13T14:06:45.510 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:06:45.510 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:06:45.510 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:06:45.510 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:06:45.630 DEBUG:teuthology.orchestra.run.smithi159:> true 2024-09-13T14:06:46.004 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:06:46.005 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T14:06:46.005 DEBUG:teuthology.orchestra.run.smithi159:> uname -r 2024-09-13T14:06:46.021 INFO:teuthology.orchestra.run.smithi159.stdout:5.14.0-505.el9.x86_64 2024-09-13T14:06:46.021 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T14:06:46.021 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T14:06:46.021 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T14:06:46.511 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T14:06:46.511 DEBUG:teuthology.orchestra.run.smithi042:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:06:46.674 INFO:teuthology.orchestra.run.smithi042.stdout:ttyS1 2024-09-13T14:06:46.714 DEBUG:teuthology.parallel:result is None 2024-09-13T14:06:47.022 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T14:06:47.023 DEBUG:teuthology.orchestra.run.smithi159:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T14:06:47.106 INFO:teuthology.orchestra.run.smithi159.stdout:ttyS1 2024-09-13T14:06:47.127 DEBUG:teuthology.parallel:result is None 2024-09-13T14:06:47.127 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T14:06:47.135 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T14:06:47.135 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:06:47.137 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T14:06:47.152 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T14:06:47.335 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T14:06:47.366 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T14:06:47.366 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:06:47.368 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T14:06:47.467 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T14:06:47.474 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T14:06:47.474 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:06:47.477 DEBUG:teuthology.orchestra.run.smithi159:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T14:06:47.509 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:06:47.523 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:06:47.711 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:06:47.722 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T14:06:47.724 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T14:06:47.731 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T14:06:47.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:06:47.734 DEBUG:teuthology.orchestra.run.smithi159:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T14:06:47.759 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T14:06:47.947 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T14:06:47.948 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:06:47.954 DEBUG:teuthology.orchestra.run.smithi159:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T14:06:47.978 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:06:48.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:06:48.184 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:06:48.262 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:06:48.326 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:06:48.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:06:48.373 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:06:48.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T14:06:48.525 DEBUG:teuthology.orchestra.run.smithi159:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:06:48.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T14:06:48.674 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:06:48.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T14:06:48.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo service rsyslog restart 2024-09-13T14:06:48.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo service rsyslog restart 2024-09-13T14:06:48.832 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:06:48.878 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T14:06:49.284 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T14:06:49.293 INFO:teuthology.task.internal:Starting timer... 2024-09-13T14:06:49.294 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T14:06:49.379 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T14:06:49.457 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-13T14:06:49.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo service auditd rotate 2024-09-13T14:06:49.647 INFO:teuthology.orchestra.run.smithi042.stdout:Rotating logs: 2024-09-13T14:06:49.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo service auditd rotate 2024-09-13T14:06:49.790 INFO:teuthology.orchestra.run.smithi159.stdout:Rotating logs: 2024-09-13T14:06:49.792 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T14:06:49.792 DEBUG:teuthology.orchestra.run.smithi042:> /usr/sbin/getenforce 2024-09-13T14:06:49.818 INFO:teuthology.orchestra.run.smithi042.stdout:Permissive 2024-09-13T14:06:49.818 DEBUG:teuthology.orchestra.run.smithi159:> /usr/sbin/getenforce 2024-09-13T14:06:49.935 INFO:teuthology.orchestra.run.smithi159.stdout:Permissive 2024-09-13T14:06:49.936 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi042.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi159.front.sepia.ceph.com': 'permissive'} 2024-09-13T14:06:49.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:06:49.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:06:49.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T14:06:49.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:06:49.999 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T14:06:49.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:06:50.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo /usr/sbin/setenforce permissive 2024-09-13T14:06:50.083 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T14:06:50.093 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T14:06:50.105 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T14:06:50.106 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi042.front.sepia.ceph.com,smithi159.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T14:10:37.626 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi042.front.sepia.ceph.com'), Remote(name='ubuntu@smithi159.front.sepia.ceph.com')] 2024-09-13T14:10:37.628 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:10:37.629 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:10:37.714 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-09-13T14:10:37.801 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T14:10:37.801 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:10:37.802 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi159.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T14:10:37.901 DEBUG:teuthology.orchestra.run.smithi159:> true 2024-09-13T14:10:37.987 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi159.front.sepia.ceph.com' 2024-09-13T14:10:37.988 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T14:10:37.998 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T14:10:37.998 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:10:37.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:10:38.001 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T14:10:38.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T14:10:38.030 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:10:38.045 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:10:38.056 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T14:10:38.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: ntpd: command not found 2024-09-13T14:10:38.071 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T14:10:38.082 INFO:teuthology.orchestra.run.smithi042.stdout:506 Cannot talk to daemon 2024-09-13T14:10:38.095 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: ntpd: command not found 2024-09-13T14:10:38.097 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:10:38.106 INFO:teuthology.orchestra.run.smithi159.stdout:506 Cannot talk to daemon 2024-09-13T14:10:38.111 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:10:38.120 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T14:10:38.134 INFO:teuthology.orchestra.run.smithi159.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T14:10:38.175 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:10:38.178 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:10:38.179 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-09-13T14:10:38.179 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.179 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.179 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.179 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.192 INFO:teuthology.orchestra.run.smithi159.stderr:bash: line 1: ntpq: command not found 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.196 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T14:10:38.197 INFO:teuthology.run_tasks:Running task pexec... 2024-09-13T14:10:38.207 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-13T14:10:38.208 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T14:10:38.208 DEBUG:teuthology.orchestra.run.smithi042:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:10:38.208 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi159.front.sepia.ceph.com 2024-09-13T14:10:38.209 DEBUG:teuthology.orchestra.run.smithi159:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-13T14:10:38.588 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:10:38.588 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:10:38.589 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Removing: 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Removing unused dependencies: 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Remove 2 Packages 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Freed space: 5.5 M 2024-09-13T14:10:38.590 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:10:38.594 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:10:38.594 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout: Package Architecture Version Repository Size 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:Removing: 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:Removing unused dependencies: 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout:Remove 2 Packages 2024-09-13T14:10:38.595 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:38.596 INFO:teuthology.orchestra.run.smithi042.stdout:Freed space: 5.5 M 2024-09-13T14:10:38.596 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:10:38.599 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:10:38.600 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:10:38.664 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:10:38.664 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:10:38.665 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:10:38.666 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:10:38.806 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:10:38.822 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:10:38.881 INFO:teuthology.orchestra.run.smithi159.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:10:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-13T14:10:38.940 INFO:teuthology.orchestra.run.smithi159.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:10:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:10:39.401 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:10:39.401 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:10:39.411 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-13T14:10:39.411 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout:Removed: 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:39.635 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout:Removed: 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:39.657 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:10:40.478 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:51 ago on Fri 13 Sep 2024 02:09:49 PM UTC. 2024-09-13T14:10:40.496 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:48 ago on Fri 13 Sep 2024 02:09:52 PM UTC. 2024-09-13T14:10:40.621 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:10:40.621 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:40.622 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:10:40.623 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:10:40.623 INFO:teuthology.orchestra.run.smithi159.stdout:Install 7 Packages 2024-09-13T14:10:40.623 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:40.630 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:10:40.633 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:40.633 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:10:40.633 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:10:40.634 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:10:40.635 INFO:teuthology.orchestra.run.smithi042.stdout:Install 7 Packages 2024-09-13T14:10:40.635 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:40.645 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 5.2 M 2024-09-13T14:10:40.645 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 19 M 2024-09-13T14:10:40.645 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:10:40.645 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 5.2 M 2024-09-13T14:10:40.646 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 19 M 2024-09-13T14:10:40.646 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:10:41.356 INFO:teuthology.orchestra.run.smithi159.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 109 kB/s | 44 kB 00:00 2024-09-13T14:10:41.391 INFO:teuthology.orchestra.run.smithi042.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 74 kB/s | 44 kB 00:00 2024-09-13T14:10:41.458 INFO:teuthology.orchestra.run.smithi042.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 160 kB/s | 106 kB 00:00 2024-09-13T14:10:41.481 INFO:teuthology.orchestra.run.smithi159.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 199 kB/s | 106 kB 00:00 2024-09-13T14:10:41.584 INFO:teuthology.orchestra.run.smithi159.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 318 kB/s | 72 kB 00:00 2024-09-13T14:10:41.607 INFO:teuthology.orchestra.run.smithi159.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 673 kB/s | 84 kB 00:00 2024-09-13T14:10:41.692 INFO:teuthology.orchestra.run.smithi042.stdout:(3/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 1.0 MB/s | 906 kB 00:00 2024-09-13T14:10:41.791 INFO:teuthology.orchestra.run.smithi159.stdout:(5/7): runc-1.1.13-4.el9.x86_64.rpm 17 MB/s | 3.2 MB 00:00 2024-09-13T14:10:41.833 INFO:teuthology.orchestra.run.smithi159.stdout:(6/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 1.0 MB/s | 906 kB 00:00 2024-09-13T14:10:42.034 INFO:teuthology.orchestra.run.smithi042.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 112 kB/s | 72 kB 00:00 2024-09-13T14:10:42.049 INFO:teuthology.orchestra.run.smithi159.stdout:(7/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 1.7 MB/s | 837 kB 00:00 2024-09-13T14:10:42.050 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:10:42.050 INFO:teuthology.orchestra.run.smithi159.stdout:Total 3.7 MB/s | 5.2 MB 00:01 2024-09-13T14:10:42.134 INFO:teuthology.orchestra.run.smithi042.stdout:(5/7): python3-kmod-0.9-32.el9.x86_64.rpm 125 kB/s | 84 kB 00:00 2024-09-13T14:10:42.159 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:10:42.174 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:10:42.174 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:10:42.454 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:10:42.454 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:10:42.458 INFO:teuthology.orchestra.run.smithi042.stdout:(6/7): runc-1.1.13-4.el9.x86_64.rpm 7.5 MB/s | 3.2 MB 00:00 2024-09-13T14:10:42.609 INFO:teuthology.orchestra.run.smithi042.stdout:(7/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 923 kB/s | 837 kB 00:00 2024-09-13T14:10:42.610 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:10:42.610 INFO:teuthology.orchestra.run.smithi042.stdout:Total 2.6 MB/s | 5.2 MB 00:01 2024-09-13T14:10:42.730 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:10:42.747 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:10:42.747 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:10:42.926 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:10:43.010 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:10:43.045 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:10:43.074 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:10:43.087 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:10:43.088 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:10:43.284 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:10:43.321 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:10:43.565 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:10:43.655 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-13T14:10:43.699 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-13T14:10:43.734 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-13T14:10:43.748 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:10:43.785 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:10:43.905 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:10:43.958 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-13T14:10:43.969 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:44.006 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:10:44.490 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-13T14:10:44.518 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:10:44.643 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-13T14:10:44.706 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:45.017 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:45.017 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:10:45.017 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:10:45.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:10:45.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:10:45.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:10:45.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:10:45.369 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:10:45.370 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:10:45.767 DEBUG:teuthology.parallel:result is None 2024-09-13T14:10:45.792 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:45.792 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-13T14:10:45.792 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-13T14:10:45.792 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-13T14:10:45.793 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-13T14:10:45.793 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-13T14:10:45.793 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-13T14:10:46.198 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:10:46.199 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:10:46.576 DEBUG:teuthology.parallel:result is None 2024-09-13T14:10:46.576 INFO:teuthology.run_tasks:Running task install... 2024-09-13T14:10:46.585 DEBUG:teuthology.task.install:project ceph 2024-09-13T14:10:46.586 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:10:46.586 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:10:46.586 INFO:teuthology.task.install:Using flavor: default 2024-09-13T14:10:46.592 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T14:10:46.592 INFO:teuthology.task.install:extra packages: [] 2024-09-13T14:10:46.593 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:10:46.593 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:10:46.593 INFO:teuthology.packaging:ref: None 2024-09-13T14:10:46.593 INFO:teuthology.packaging:tag: None 2024-09-13T14:10:46.593 INFO:teuthology.packaging:branch: reef 2024-09-13T14:10:46.593 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:10:46.594 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-13T14:10:46.596 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'tag': None, 'wait_for_package': False} 2024-09-13T14:10:46.596 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:10:46.596 INFO:teuthology.packaging:ref: None 2024-09-13T14:10:46.596 INFO:teuthology.packaging:tag: None 2024-09-13T14:10:46.596 INFO:teuthology.packaging:branch: reef 2024-09-13T14:10:46.596 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:10:46.596 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-13T14:10:46.853 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/ 2024-09-13T14:10:46.854 INFO:teuthology.task.install.rpm:Package version is 18.2.4-843.g2c01458a 2024-09-13T14:10:46.888 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/ 2024-09-13T14:10:46.888 INFO:teuthology.task.install.rpm:Package version is 18.2.4-843.g2c01458a 2024-09-13T14:10:46.990 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:10:46.991 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:10:46.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:10:47.005 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-13T14:10:47.005 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:10:47.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-13T14:10:47.022 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:10:47.022 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:10:47.022 INFO:teuthology.packaging:ref: None 2024-09-13T14:10:47.022 INFO:teuthology.packaging:tag: None 2024-09-13T14:10:47.022 INFO:teuthology.packaging:branch: reef 2024-09-13T14:10:47.022 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:10:47.023 DEBUG:teuthology.orchestra.run.smithi159:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:10:47.041 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-13T14:10:47.041 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:10:47.041 INFO:teuthology.packaging:ref: None 2024-09-13T14:10:47.041 INFO:teuthology.packaging:tag: None 2024-09-13T14:10:47.041 INFO:teuthology.packaging:branch: reef 2024-09-13T14:10:47.041 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:10:47.042 DEBUG:teuthology.orchestra.run.smithi042:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-13T14:10:47.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:10:47.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-13T14:10:47.173 DEBUG:teuthology.orchestra.run.smithi159:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:10:47.191 DEBUG:teuthology.orchestra.run.smithi042:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-13T14:10:47.239 INFO:teuthology.orchestra.run.smithi159.stdout:check_obsoletes = 1 2024-09-13T14:10:47.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum clean all 2024-09-13T14:10:47.256 INFO:teuthology.orchestra.run.smithi042.stdout:check_obsoletes = 1 2024-09-13T14:10:47.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum clean all 2024-09-13T14:10:47.485 INFO:teuthology.orchestra.run.smithi159.stdout:45 files removed 2024-09-13T14:10:47.488 INFO:teuthology.orchestra.run.smithi042.stdout:45 files removed 2024-09-13T14:10:47.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-radosgw 2024-09-13T14:10:47.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-radosgw 2024-09-13T14:10:48.065 INFO:teuthology.orchestra.run.smithi159.stdout:ceph packages for x86_64 289 kB/s | 77 kB 00:00 2024-09-13T14:10:48.082 INFO:teuthology.orchestra.run.smithi042.stdout:ceph packages for x86_64 287 kB/s | 77 kB 00:00 2024-09-13T14:10:48.349 INFO:teuthology.orchestra.run.smithi159.stdout:ceph noarch packages 47 kB/s | 11 kB 00:00 2024-09-13T14:10:48.383 INFO:teuthology.orchestra.run.smithi042.stdout:ceph noarch packages 46 kB/s | 11 kB 00:00 2024-09-13T14:10:48.635 INFO:teuthology.orchestra.run.smithi159.stdout:ceph source packages 7.6 kB/s | 1.9 kB 00:00 2024-09-13T14:10:48.659 INFO:teuthology.orchestra.run.smithi042.stdout:ceph source packages 8.0 kB/s | 1.9 kB 00:00 2024-09-13T14:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - BaseOS 5.2 MB/s | 8.2 MB 00:01 2024-09-13T14:10:51.509 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - BaseOS 2.9 MB/s | 8.2 MB 00:02 2024-09-13T14:10:53.601 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - AppStream 23 MB/s | 20 MB 00:00 2024-09-13T14:11:00.102 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - CRB 2.7 MB/s | 6.5 MB 00:02 2024-09-13T14:11:01.819 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - Extras packages 44 kB/s | 19 kB 00:00 2024-09-13T14:11:03.232 INFO:teuthology.orchestra.run.smithi042.stdout:Extra Packages for Enterprise Linux 18 MB/s | 23 MB 00:01 2024-09-13T14:11:08.954 INFO:teuthology.orchestra.run.smithi042.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-09-13T14:11:10.715 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 20 MB 00:19 2024-09-13T14:11:11.405 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:11:11.407 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:11.407 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:11:11.407 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-radosgw x86_64 2:18.2.4-843.g2c01458a.el9 ceph 7.7 M 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout:Upgrading: 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: librados2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.3 M 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: librbd1 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.0 M 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-base x86_64 2:18.2.4-843.g2c01458a.el9 ceph 5.1 M 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-common x86_64 2:18.2.4-843.g2c01458a.el9 ceph 18 M 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-selinux x86_64 2:18.2.4-843.g2c01458a.el9 ceph 25 k 2024-09-13T14:11:11.408 INFO:teuthology.orchestra.run.smithi042.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libcephfs2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 712 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libradosstriper1 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 477 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: librgw2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 4.5 M 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:11:11.409 INFO:teuthology.orchestra.run.smithi042.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ceph-argparse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 45 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ceph-common x86_64 2:18.2.4-843.g2c01458a.el9 ceph 129 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cephfs x86_64 2:18.2.4-843.g2c01458a.el9 ceph 162 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rados x86_64 2:18.2.4-843.g2c01458a.el9 ceph 321 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rbd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 301 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rgw x86_64 2:18.2.4-843.g2c01458a.el9 ceph 99 k 2024-09-13T14:11:11.410 INFO:teuthology.orchestra.run.smithi042.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout:Install 28 Packages 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout:Upgrade 2 Packages 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:11.411 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 53 M 2024-09-13T14:11:11.412 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:11:12.156 INFO:teuthology.orchestra.run.smithi042.stdout:(1/30): ceph-radosgw-18.2.4-843.g2c01458a.el9.x 11 MB/s | 7.7 MB 00:00 2024-09-13T14:11:12.231 INFO:teuthology.orchestra.run.smithi042.stdout:(2/30): ceph-base-18.2.4-843.g2c01458a.el9.x86_ 6.6 MB/s | 5.1 MB 00:00 2024-09-13T14:11:12.256 INFO:teuthology.orchestra.run.smithi042.stdout:(3/30): ceph-selinux-18.2.4-843.g2c01458a.el9.x 251 kB/s | 25 kB 00:00 2024-09-13T14:11:12.315 INFO:teuthology.orchestra.run.smithi042.stdout:(4/30): libcephfs2-18.2.4-843.g2c01458a.el9.x86 8.3 MB/s | 712 kB 00:00 2024-09-13T14:11:12.349 INFO:teuthology.orchestra.run.smithi042.stdout:(5/30): libradosstriper1-18.2.4-843.g2c01458a.e 5.1 MB/s | 477 kB 00:00 2024-09-13T14:11:12.557 INFO:teuthology.orchestra.run.smithi042.stdout:(6/30): ceph-common-18.2.4-843.g2c01458a.el9.x8 17 MB/s | 18 MB 00:01 2024-09-13T14:11:12.582 INFO:teuthology.orchestra.run.smithi042.stdout:(7/30): python3-ceph-argparse-18.2.4-843.g2c014 193 kB/s | 45 kB 00:00 2024-09-13T14:11:12.608 INFO:teuthology.orchestra.run.smithi042.stdout:(8/30): python3-ceph-common-18.2.4-843.g2c01458 2.5 MB/s | 129 kB 00:00 2024-09-13T14:11:12.633 INFO:teuthology.orchestra.run.smithi042.stdout:(9/30): python3-cephfs-18.2.4-843.g2c01458a.el9 3.2 MB/s | 162 kB 00:00 2024-09-13T14:11:12.658 INFO:teuthology.orchestra.run.smithi042.stdout:(10/30): python3-rados-18.2.4-843.g2c01458a.el9 6.3 MB/s | 321 kB 00:00 2024-09-13T14:11:12.683 INFO:teuthology.orchestra.run.smithi042.stdout:(11/30): python3-rbd-18.2.4-843.g2c01458a.el9.x 5.8 MB/s | 301 kB 00:00 2024-09-13T14:11:12.709 INFO:teuthology.orchestra.run.smithi042.stdout:(12/30): python3-rgw-18.2.4-843.g2c01458a.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-13T14:11:12.784 INFO:teuthology.orchestra.run.smithi042.stdout:(13/30): librgw2-18.2.4-843.g2c01458a.el9.x86_6 9.5 MB/s | 4.5 MB 00:00 2024-09-13T14:11:12.826 INFO:teuthology.orchestra.run.smithi042.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 752 kB/s | 107 kB 00:00 2024-09-13T14:11:12.876 INFO:teuthology.orchestra.run.smithi042.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 493 kB/s | 45 kB 00:00 2024-09-13T14:11:12.901 INFO:teuthology.orchestra.run.smithi042.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 832 kB/s | 160 kB 00:00 2024-09-13T14:11:12.926 INFO:teuthology.orchestra.run.smithi042.stdout:(17/30): librdkafka-1.6.1-102.el9.x86_64.rpm 6.5 MB/s | 662 kB 00:00 2024-09-13T14:11:12.952 INFO:teuthology.orchestra.run.smithi042.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 840 kB/s | 42 kB 00:00 2024-09-13T14:11:12.994 INFO:teuthology.orchestra.run.smithi042.stdout:(19/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 292 kB 00:00 2024-09-13T14:11:13.027 INFO:teuthology.orchestra.run.smithi042.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 1.1 MB/s | 111 kB 00:00 2024-09-13T14:11:13.053 INFO:teuthology.orchestra.run.smithi042.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-09-13T14:11:13.078 INFO:teuthology.orchestra.run.smithi042.stdout:(22/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.4 MB/s | 308 kB 00:00 2024-09-13T14:11:13.095 INFO:teuthology.orchestra.run.smithi042.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 1.1 MB/s | 49 kB 00:00 2024-09-13T14:11:13.120 INFO:teuthology.orchestra.run.smithi042.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.6 MB/s | 67 kB 00:00 2024-09-13T14:11:13.162 INFO:teuthology.orchestra.run.smithi042.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 4.5 MB/s | 191 kB 00:00 2024-09-13T14:11:13.196 INFO:teuthology.orchestra.run.smithi042.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.2 MB/s | 839 kB 00:00 2024-09-13T14:11:13.329 INFO:teuthology.orchestra.run.smithi042.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 9.5 MB/s | 1.6 MB 00:00 2024-09-13T14:11:13.421 INFO:teuthology.orchestra.run.smithi042.stdout:(28/30): librados2-18.2.4-843.g2c01458a.el9.x86 15 MB/s | 3.3 MB 00:00 2024-09-13T14:11:13.505 INFO:teuthology.orchestra.run.smithi042.stdout:(29/30): libarrow-9.0.0-11.el9.x86_64.rpm 8.7 MB/s | 4.4 MB 00:00 2024-09-13T14:11:13.563 INFO:teuthology.orchestra.run.smithi042.stdout:(30/30): librbd1-18.2.4-843.g2c01458a.el9.x86_6 13 MB/s | 3.0 MB 00:00 2024-09-13T14:11:13.568 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:13.568 INFO:teuthology.orchestra.run.smithi042.stdout:Total 25 MB/s | 53 MB 00:02 2024-09-13T14:11:13.638 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:11:13.677 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:11:13.677 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:11:14.169 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:11:14.170 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:11:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:11:15.747 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-13T14:11:15.777 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-13T14:11:15.836 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-13T14:11:15.941 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 6.5 MB 00:01 2024-09-13T14:11:16.165 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-13T14:11:16.285 INFO:teuthology.orchestra.run.smithi042.stdout: Upgrading : librados2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:16.377 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librados2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:16.472 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 6/32 2024-09-13T14:11:16.620 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:16.682 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:16.711 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-13T14:11:16.755 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x 9/32 2024-09-13T14:11:16.823 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 10/32 2024-09-13T14:11:16.838 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:16.921 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:17.040 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-13T14:11:17.080 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-13T14:11:17.131 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-13T14:11:17.161 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-13T14:11:17.188 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-13T14:11:17.224 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-13T14:11:17.257 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-13T14:11:17.458 INFO:teuthology.orchestra.run.smithi159.stdout:CentOS Stream 9 - Extras packages 62 kB/s | 19 kB 00:00 2024-09-13T14:11:17.572 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-13T14:11:17.611 INFO:teuthology.orchestra.run.smithi042.stdout: Upgrading : librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 20/32 2024-09-13T14:11:17.703 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 20/32 2024-09-13T14:11:17.733 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 21/32 2024-09-13T14:11:17.778 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-13T14:11:18.219 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-13T14:11:18.255 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 24/32 2024-09-13T14:11:18.319 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 24/32 2024-09-13T14:11:18.398 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 25/32 2024-09-13T14:11:18.549 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86 26/32 2024-09-13T14:11:18.720 INFO:teuthology.orchestra.run.smithi159.stdout:Extra Packages for Enterprise Linux 20 MB/s | 23 MB 00:01 2024-09-13T14:11:20.718 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:20.754 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 28/32 2024-09-13T14:11:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 28/32 2024-09-13T14:11:21.470 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:11:21.470 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:11:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:21.517 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:24.077 INFO:teuthology.orchestra.run.smithi159.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-09-13T14:11:26.481 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-radosgw x86_64 2:18.2.4-843.g2c01458a.el9 ceph 7.7 M 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout:Upgrading: 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: librados2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.3 M 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: librbd1 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.0 M 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-base x86_64 2:18.2.4-843.g2c01458a.el9 ceph 5.1 M 2024-09-13T14:11:26.483 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-common x86_64 2:18.2.4-843.g2c01458a.el9 ceph 18 M 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-selinux x86_64 2:18.2.4-843.g2c01458a.el9 ceph 25 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 712 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-13T14:11:26.484 INFO:teuthology.orchestra.run.smithi159.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: libradosstriper1 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 477 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: librgw2 x86_64 2:18.2.4-843.g2c01458a.el9 ceph 4.5 M 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-argparse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 45 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-common x86_64 2:18.2.4-843.g2c01458a.el9 ceph 129 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cephfs x86_64 2:18.2.4-843.g2c01458a.el9 ceph 162 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rados x86_64 2:18.2.4-843.g2c01458a.el9 ceph 321 k 2024-09-13T14:11:26.485 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rbd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 301 k 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rgw x86_64 2:18.2.4-843.g2c01458a.el9 ceph 99 k 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout:Install 28 Packages 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout:Upgrade 2 Packages 2024-09-13T14:11:26.486 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:26.487 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 53 M 2024-09-13T14:11:26.487 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:11:27.370 INFO:teuthology.orchestra.run.smithi159.stdout:(1/30): ceph-base-18.2.4-843.g2c01458a.el9.x86_ 11 MB/s | 5.1 MB 00:00 2024-09-13T14:11:27.437 INFO:teuthology.orchestra.run.smithi159.stdout:(2/30): ceph-selinux-18.2.4-843.g2c01458a.el9.x 378 kB/s | 25 kB 00:00 2024-09-13T14:11:27.520 INFO:teuthology.orchestra.run.smithi159.stdout:(3/30): ceph-radosgw-18.2.4-843.g2c01458a.el9.x 13 MB/s | 7.7 MB 00:00 2024-09-13T14:11:27.566 INFO:teuthology.orchestra.run.smithi159.stdout:(4/30): libcephfs2-18.2.4-843.g2c01458a.el9.x86 5.6 MB/s | 712 kB 00:00 2024-09-13T14:11:27.588 INFO:teuthology.orchestra.run.smithi159.stdout:(5/30): libradosstriper1-18.2.4-843.g2c01458a.e 7.0 MB/s | 477 kB 00:00 2024-09-13T14:11:27.646 INFO:teuthology.orchestra.run.smithi159.stdout:(6/30): python3-ceph-argparse-18.2.4-843.g2c014 772 kB/s | 45 kB 00:00 2024-09-13T14:11:27.771 INFO:teuthology.orchestra.run.smithi159.stdout:(7/30): librgw2-18.2.4-843.g2c01458a.el9.x86_64 21 MB/s | 4.5 MB 00:00 2024-09-13T14:11:27.797 INFO:teuthology.orchestra.run.smithi159.stdout:(8/30): python3-ceph-common-18.2.4-843.g2c01458 861 kB/s | 129 kB 00:00 2024-09-13T14:11:27.822 INFO:teuthology.orchestra.run.smithi159.stdout:(9/30): python3-cephfs-18.2.4-843.g2c01458a.el9 3.2 MB/s | 162 kB 00:00 2024-09-13T14:11:27.847 INFO:teuthology.orchestra.run.smithi159.stdout:(10/30): python3-rados-18.2.4-843.g2c01458a.el9 6.3 MB/s | 321 kB 00:00 2024-09-13T14:11:27.881 INFO:teuthology.orchestra.run.smithi159.stdout:(11/30): python3-rbd-18.2.4-843.g2c01458a.el9.x 5.0 MB/s | 301 kB 00:00 2024-09-13T14:11:27.906 INFO:teuthology.orchestra.run.smithi159.stdout:(12/30): python3-rgw-18.2.4-843.g2c01458a.el9.x 1.7 MB/s | 99 kB 00:00 2024-09-13T14:11:28.098 INFO:teuthology.orchestra.run.smithi159.stdout:(13/30): ceph-common-18.2.4-843.g2c01458a.el9.x 15 MB/s | 18 MB 00:01 2024-09-13T14:11:28.673 INFO:teuthology.orchestra.run.smithi159.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 79 kB/s | 45 kB 00:00 2024-09-13T14:11:28.698 INFO:teuthology.orchestra.run.smithi159.stdout:(15/30): boost-program-options-1.75.0-8.el9.x86 131 kB/s | 107 kB 00:00 2024-09-13T14:11:28.741 INFO:teuthology.orchestra.run.smithi159.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 192 kB/s | 160 kB 00:00 2024-09-13T14:11:29.308 INFO:teuthology.orchestra.run.smithi159.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 75 kB/s | 42 kB 00:00 2024-09-13T14:11:29.358 INFO:teuthology.orchestra.run.smithi159.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 2.2 MB/s | 111 kB 00:00 2024-09-13T14:11:29.391 INFO:teuthology.orchestra.run.smithi159.stdout:(19/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 9.0 MB/s | 308 kB 00:00 2024-09-13T14:11:29.534 INFO:teuthology.orchestra.run.smithi159.stdout:(20/30): librdkafka-1.6.1-102.el9.x86_64.rpm 771 kB/s | 662 kB 00:00 2024-09-13T14:11:29.558 INFO:teuthology.orchestra.run.smithi159.stdout:(21/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 340 kB/s | 292 kB 00:00 2024-09-13T14:11:29.600 INFO:teuthology.orchestra.run.smithi159.stdout:(22/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 391 kB/s | 26 kB 00:00 2024-09-13T14:11:29.626 INFO:teuthology.orchestra.run.smithi159.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 820 kB/s | 49 kB 00:00 2024-09-13T14:11:29.651 INFO:teuthology.orchestra.run.smithi159.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.4 MB/s | 67 kB 00:00 2024-09-13T14:11:29.718 INFO:teuthology.orchestra.run.smithi159.stdout:(25/30): libarrow-9.0.0-11.el9.x86_64.rpm 14 MB/s | 4.4 MB 00:00 2024-09-13T14:11:29.743 INFO:teuthology.orchestra.run.smithi159.stdout:(26/30): re2-20211101-20.el9.x86_64.rpm 2.0 MB/s | 191 kB 00:00 2024-09-13T14:11:29.802 INFO:teuthology.orchestra.run.smithi159.stdout:(27/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 4.6 MB/s | 839 kB 00:00 2024-09-13T14:11:29.844 INFO:teuthology.orchestra.run.smithi159.stdout:(28/30): thrift-0.15.0-2.el9.x86_64.rpm 13 MB/s | 1.6 MB 00:00 2024-09-13T14:11:30.020 INFO:teuthology.orchestra.run.smithi159.stdout:(29/30): librados2-18.2.4-843.g2c01458a.el9.x86 12 MB/s | 3.3 MB 00:00 2024-09-13T14:11:30.086 INFO:teuthology.orchestra.run.smithi159.stdout:(30/30): librbd1-18.2.4-843.g2c01458a.el9.x86_6 11 MB/s | 3.0 MB 00:00 2024-09-13T14:11:30.092 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:30.092 INFO:teuthology.orchestra.run.smithi159.stdout:Total 15 MB/s | 53 MB 00:03 2024-09-13T14:11:30.162 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:11:30.198 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:11:30.199 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:11:30.665 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:11:30.665 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /sys 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /proc 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /mnt 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /var/tmp 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /home 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /root 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout:skipping the directory /tmp 2024-09-13T14:11:32.116 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:11:32.211 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-13T14:11:32.242 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-13T14:11:32.299 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-13T14:11:32.677 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-13T14:11:32.732 INFO:teuthology.orchestra.run.smithi159.stdout: Upgrading : librados2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:32.833 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librados2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:32.900 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 30/32 2024-09-13T14:11:32.916 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 6/32 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 30/32 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:11:32.934 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:32.936 INFO:teuthology.orchestra.run.smithi042.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-13T14:11:32.952 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:33.004 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-13T14:11:33.004 INFO:teuthology.orchestra.run.smithi042.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:33.015 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:33.051 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-13T14:11:33.095 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x 9/32 2024-09-13T14:11:33.161 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 10/32 2024-09-13T14:11:33.187 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:33.281 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:33.383 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-13T14:11:33.415 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-13T14:11:33.471 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-13T14:11:33.510 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-13T14:11:33.537 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-13T14:11:33.573 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-13T14:11:33.606 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-13T14:11:33.906 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-13T14:11:33.944 INFO:teuthology.orchestra.run.smithi159.stdout: Upgrading : librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 20/32 2024-09-13T14:11:34.034 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 20/32 2024-09-13T14:11:34.065 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 21/32 2024-09-13T14:11:34.101 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-13T14:11:34.145 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:34.145 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 1/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 3/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 4/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 6/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x 8/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86 9/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 10/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 12/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 13/32 2024-09-13T14:11:34.146 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-13T14:11:34.147 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-13T14:11:34.148 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-13T14:11:34.148 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librados2-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:34.148 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-13T14:11:34.148 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 31/32 2024-09-13T14:11:34.507 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-13T14:11:34.546 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 24/32 2024-09-13T14:11:34.617 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 24/32 2024-09-13T14:11:34.703 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 25/32 2024-09-13T14:11:34.741 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86 26/32 2024-09-13T14:11:35.139 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:35.139 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:35.139 INFO:teuthology.orchestra.run.smithi042.stdout:Upgraded: 2024-09-13T14:11:35.139 INFO:teuthology.orchestra.run.smithi042.stdout: librados2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.139 INFO:teuthology.orchestra.run.smithi042.stdout: librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-13T14:11:35.140 INFO:teuthology.orchestra.run.smithi042.stdout: libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-13T14:11:35.141 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: re2-1:20211101-20.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:35.142 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:11:35.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-test 2024-09-13T14:11:36.078 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:28 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:11:36.223 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-test x86_64 2:18.2.4-843.g2c01458a.el9 ceph 40 M 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:11:36.224 INFO:teuthology.orchestra.run.smithi042.stdout: libcephsqlite x86_64 2:18.2.4-843.g2c01458a.el9 ceph 166 k 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout:Install 4 Packages 2024-09-13T14:11:36.225 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:36.226 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 40 M 2024-09-13T14:11:36.226 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 164 M 2024-09-13T14:11:36.226 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:11:36.576 INFO:teuthology.orchestra.run.smithi042.stdout:(1/4): libcephsqlite-18.2.4-843.g2c01458a.el9.x 747 kB/s | 166 kB 00:00 2024-09-13T14:11:36.920 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:36.943 INFO:teuthology.orchestra.run.smithi042.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 174 kB/s | 64 kB 00:00 2024-09-13T14:11:36.954 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:36.993 INFO:teuthology.orchestra.run.smithi042.stdout:(3/4): socat-1.7.4.1-6.el9.x86_64.rpm 476 kB/s | 304 kB 00:00 2024-09-13T14:11:37.494 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 27/32 2024-09-13T14:11:37.577 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 28/32 2024-09-13T14:11:37.673 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 28/32 2024-09-13T14:11:37.673 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-13T14:11:37.673 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-13T14:11:37.673 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:37.685 INFO:teuthology.orchestra.run.smithi042.stdout:(4/4): ceph-test-18.2.4-843.g2c01458a.el9.x86_6 30 MB/s | 40 MB 00:01 2024-09-13T14:11:37.687 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:37.688 INFO:teuthology.orchestra.run.smithi042.stdout:Total 27 MB/s | 40 MB 00:01 2024-09-13T14:11:37.708 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:37.716 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:11:37.738 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:11:37.738 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:11:38.078 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:11:38.078 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:11:38.584 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:11:38.663 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:11:38.702 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:11:38.729 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 3/4 2024-09-13T14:11:42.654 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 3/4 2024-09-13T14:11:42.739 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 4/4 2024-09-13T14:11:44.700 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 4/4 2024-09-13T14:11:44.700 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 1/4 2024-09-13T14:11:44.700 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 2/4 2024-09-13T14:11:44.700 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:11:45.007 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:45.008 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:11:45.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph 2024-09-13T14:11:45.881 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:37 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:11:46.026 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout: ceph x86_64 2:18.2.4-843.g2c01458a.el9 ceph 6.5 k 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:11:46.028 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mds x86_64 2:18.2.4-843.g2c01458a.el9 ceph 2.1 M 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr x86_64 2:18.2.4-843.g2c01458a.el9 ceph 1.5 M 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-modules-core noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 246 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mon x86_64 2:18.2.4-843.g2c01458a.el9 ceph 4.7 M 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-osd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 17 M 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:11:46.029 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:11:46.030 INFO:teuthology.orchestra.run.smithi042.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout:Install 33 Packages 2024-09-13T14:11:46.031 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:46.032 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 29 M 2024-09-13T14:11:46.032 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 104 M 2024-09-13T14:11:46.032 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:11:46.571 INFO:teuthology.orchestra.run.smithi042.stdout:(1/33): ceph-18.2.4-843.g2c01458a.el9.x86_64.rp 50 kB/s | 6.5 kB 00:00 2024-09-13T14:11:46.779 INFO:teuthology.orchestra.run.smithi042.stdout:(2/33): ceph-mds-18.2.4-843.g2c01458a.el9.x86_6 6.2 MB/s | 2.1 MB 00:00 2024-09-13T14:11:46.821 INFO:teuthology.orchestra.run.smithi042.stdout:(3/33): ceph-mgr-18.2.4-843.g2c01458a.el9.x86_6 3.8 MB/s | 1.5 MB 00:00 2024-09-13T14:11:46.913 INFO:teuthology.orchestra.run.smithi042.stdout:(4/33): ceph-mgr-modules-core-18.2.4-843.g2c014 2.6 MB/s | 246 kB 00:00 2024-09-13T14:11:47.022 INFO:teuthology.orchestra.run.smithi042.stdout:(5/33): ceph-mon-18.2.4-843.g2c01458a.el9.x86_6 10 MB/s | 4.7 MB 00:00 2024-09-13T14:11:47.089 INFO:teuthology.orchestra.run.smithi042.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-09-13T14:11:47.131 INFO:teuthology.orchestra.run.smithi042.stdout:(7/33): python3-ply-3.11-14.el9.noarch.rpm 2.5 MB/s | 106 kB 00:00 2024-09-13T14:11:47.173 INFO:teuthology.orchestra.run.smithi042.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 3.2 MB/s | 135 kB 00:00 2024-09-13T14:11:47.348 INFO:teuthology.orchestra.run.smithi042.stdout:(9/33): ceph-osd-18.2.4-843.g2c01458a.el9.x86_6 29 MB/s | 17 MB 00:00 2024-09-13T14:11:47.373 INFO:teuthology.orchestra.run.smithi042.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 623 kB/s | 125 kB 00:00 2024-09-13T14:11:47.415 INFO:teuthology.orchestra.run.smithi042.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 3.1 MB/s | 1.2 MB 00:00 2024-09-13T14:11:47.440 INFO:teuthology.orchestra.run.smithi042.stdout:(12/33): python3-mako-1.1.4-6.el9.noarch.rpm 2.5 MB/s | 172 kB 00:00 2024-09-13T14:11:47.466 INFO:teuthology.orchestra.run.smithi042.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 691 kB/s | 35 kB 00:00 2024-09-13T14:11:47.491 INFO:teuthology.orchestra.run.smithi042.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.0 MB/s | 51 kB 00:00 2024-09-13T14:11:47.516 INFO:teuthology.orchestra.run.smithi042.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 831 kB/s | 42 kB 00:00 2024-09-13T14:11:47.541 INFO:teuthology.orchestra.run.smithi042.stdout:(16/33): python3-urllib3-1.26.5-6.el9.noarch.rp 1.1 MB/s | 215 kB 00:00 2024-09-13T14:11:47.592 INFO:teuthology.orchestra.run.smithi042.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 432 kB/s | 43 kB 00:00 2024-09-13T14:11:47.617 INFO:teuthology.orchestra.run.smithi042.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 468 kB/s | 11 kB 00:00 2024-09-13T14:11:47.642 INFO:teuthology.orchestra.run.smithi042.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-13T14:11:47.668 INFO:teuthology.orchestra.run.smithi042.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-09-13T14:11:47.693 INFO:teuthology.orchestra.run.smithi042.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 462 kB/s | 23 kB 00:00 2024-09-13T14:11:47.710 INFO:teuthology.orchestra.run.smithi042.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 467 kB/s | 19 kB 00:00 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /sys 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /proc 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /mnt 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /var/tmp 2024-09-13T14:11:47.721 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /home 2024-09-13T14:11:47.722 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /root 2024-09-13T14:11:47.722 INFO:teuthology.orchestra.run.smithi159.stdout:skipping the directory /tmp 2024-09-13T14:11:47.722 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:47.727 INFO:teuthology.orchestra.run.smithi042.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 587 kB/s | 20 kB 00:00 2024-09-13T14:11:47.744 INFO:teuthology.orchestra.run.smithi042.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.3 MB/s | 46 kB 00:00 2024-09-13T14:11:47.769 INFO:teuthology.orchestra.run.smithi042.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.8 MB/s | 79 kB 00:00 2024-09-13T14:11:47.794 INFO:teuthology.orchestra.run.smithi042.stdout:(26/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.5 MB/s | 358 kB 00:00 2024-09-13T14:11:47.811 INFO:teuthology.orchestra.run.smithi042.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 392 kB/s | 16 kB 00:00 2024-09-13T14:11:47.836 INFO:teuthology.orchestra.run.smithi042.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.1 MB/s | 90 kB 00:00 2024-09-13T14:11:47.861 INFO:teuthology.orchestra.run.smithi042.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 716 kB/s | 36 kB 00:00 2024-09-13T14:11:47.887 INFO:teuthology.orchestra.run.smithi042.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.9 MB/s | 272 kB 00:00 2024-09-13T14:11:47.912 INFO:teuthology.orchestra.run.smithi042.stdout:(31/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-13T14:11:47.937 INFO:teuthology.orchestra.run.smithi042.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 399 kB/s | 20 kB 00:00 2024-09-13T14:11:47.962 INFO:teuthology.orchestra.run.smithi042.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.2 MB/s | 427 kB 00:00 2024-09-13T14:11:47.967 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:47.967 INFO:teuthology.orchestra.run.smithi042.stdout:Total 15 MB/s | 29 MB 00:01 2024-09-13T14:11:48.096 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:11:48.138 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:11:48.138 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:11:48.490 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 30/32 2024-09-13T14:11:48.521 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 30/32 2024-09-13T14:11:48.521 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:48.521 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-13T14:11:48.522 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:11:48.522 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-13T14:11:48.522 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:48.523 INFO:teuthology.orchestra.run.smithi159.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-13T14:11:48.548 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:11:48.548 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:11:48.610 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-13T14:11:48.610 INFO:teuthology.orchestra.run.smithi159.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:49.731 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:49.731 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 1/32 2024-09-13T14:11:49.731 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 3/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 4/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 5/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 6/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 7/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x 8/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86 9/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 10/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 11/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 12/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 13/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-13T14:11:49.732 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-13T14:11:49.733 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-13T14:11:49.734 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados2-2:18.2.4-843.g2c01458a.el9.x86_64 29/32 2024-09-13T14:11:49.734 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-13T14:11:49.734 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 31/32 2024-09-13T14:11:49.884 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:11:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-13T14:11:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-13T14:11:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-13T14:11:50.145 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-13T14:11:50.221 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-13T14:11:50.277 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-13T14:11:50.319 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-13T14:11:50.349 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-13T14:11:50.384 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-13T14:11:50.416 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-13T14:11:50.466 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-13T14:11:50.509 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-13T14:11:50.543 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-13T14:11:50.588 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-13T14:11:50.667 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-13T14:11:50.785 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout:Upgraded: 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: librados2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-radosgw-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-selinux-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-13T14:11:50.786 INFO:teuthology.orchestra.run.smithi159.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: libradosstriper1-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: librgw2-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-argparse-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.787 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ceph-common-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: re2-1:20211101-20.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:50.788 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:11:50.957 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-13T14:11:51.045 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-13T14:11:51.096 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-13T14:11:51.133 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-13T14:11:51.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-test 2024-09-13T14:11:51.191 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-13T14:11:51.256 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-13T14:11:51.460 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-13T14:11:51.520 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-13T14:11:51.601 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-13T14:11:51.692 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-13T14:11:51.730 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-13T14:11:51.754 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:27 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:11:51.849 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-test x86_64 2:18.2.4-843.g2c01458a.el9 ceph 40 M 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout: libcephsqlite x86_64 2:18.2.4-843.g2c01458a.el9 ceph 166 k 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-13T14:11:51.901 INFO:teuthology.orchestra.run.smithi159.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:Install 4 Packages 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 40 M 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 164 M 2024-09-13T14:11:51.902 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:11:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.n 28/33 2024-09-13T14:11:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 29/33 2024-09-13T14:11:52.072 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 29/33 2024-09-13T14:11:52.072 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:52.073 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:11:52.073 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:11:52.073 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:11:52.073 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:52.085 INFO:teuthology.orchestra.run.smithi159.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.9 MB/s | 304 kB 00:00 2024-09-13T14:11:52.127 INFO:teuthology.orchestra.run.smithi159.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-09-13T14:11:52.152 INFO:teuthology.orchestra.run.smithi159.stdout:(3/4): libcephsqlite-18.2.4-843.g2c01458a.el9.x 733 kB/s | 166 kB 00:00 2024-09-13T14:11:53.211 INFO:teuthology.orchestra.run.smithi159.stdout:(4/4): ceph-test-18.2.4-843.g2c01458a.el9.x86_6 31 MB/s | 40 MB 00:01 2024-09-13T14:11:53.213 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:53.213 INFO:teuthology.orchestra.run.smithi159.stdout:Total 31 MB/s | 40 MB 00:01 2024-09-13T14:11:53.242 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:11:53.261 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:11:53.261 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:11:53.580 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 30/33 2024-09-13T14:11:53.588 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:11:53.588 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:11:53.611 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 30/33 2024-09-13T14:11:53.612 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:53.612 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:11:53.612 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:11:53.612 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:11:53.612 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:54.057 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:11:54.072 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 31/33 2024-09-13T14:11:54.098 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 31/33 2024-09-13T14:11:54.098 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:54.098 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:11:54.099 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:11:54.099 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:11:54.099 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:54.176 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-13T14:11:54.227 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-13T14:11:54.263 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 3/4 2024-09-13T14:11:54.307 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 32/33 2024-09-13T14:11:54.331 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 32/33 2024-09-13T14:11:54.332 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:11:54.332 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:11:54.332 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:11:54.332 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:11:54.332 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:54.398 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-2:18.2.4-843.g2c01458a.el9.x86_64 33/33 2024-09-13T14:11:56.198 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-2:18.2.4-843.g2c01458a.el9.x86_64 33/33 2024-09-13T14:11:56.198 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-2:18.2.4-843.g2c01458a.el9.x86_64 1/33 2024-09-13T14:11:56.198 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 2/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 3/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 4/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 5/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.n 6/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-13T14:11:56.199 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-13T14:11:56.200 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-13T14:11:56.201 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-13T14:11:56.201 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-13T14:11:56.201 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-13T14:11:56.201 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-13T14:11:57.242 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:11:57.243 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:11:57.244 INFO:teuthology.orchestra.run.smithi042.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:57.245 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:11:57.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-base 2024-09-13T14:11:57.969 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 3/4 2024-09-13T14:11:58.060 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 4/4 2024-09-13T14:11:58.247 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:50 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:11:58.341 INFO:teuthology.orchestra.run.smithi042.stdout:Package ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:11:58.385 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:11:58.386 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:11:58.386 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:11:58.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install cephadm 2024-09-13T14:11:59.045 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:51 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:11:59.184 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout: cephadm noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 226 k 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:11:59.185 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:11:59.186 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:11:59.186 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 226 k 2024-09-13T14:11:59.186 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 217 k 2024-09-13T14:11:59.186 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:11:59.430 INFO:teuthology.orchestra.run.smithi042.stdout:cephadm-18.2.4-843.g2c01458a.el9.noarch.rpm 925 kB/s | 226 kB 00:00 2024-09-13T14:11:59.431 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:11:59.431 INFO:teuthology.orchestra.run.smithi042.stdout:Total 919 kB/s | 226 kB 00:00 2024-09-13T14:11:59.431 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:11:59.437 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:11:59.437 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:11:59.440 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:11:59.440 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:11:59.512 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:11:59.886 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:11:59.971 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:00.019 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 4/4 2024-09-13T14:12:00.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 1/4 2024-09-13T14:12:00.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 2/4 2024-09-13T14:12:00.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-13T14:12:00.295 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:00.350 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-13T14:12:00.350 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-test-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout: libcephsqlite-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:00.351 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:00.559 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:00.560 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:00.672 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph 2024-09-13T14:12:00.790 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:12:01.242 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:37 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:01.377 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:53 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:01.379 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:01.381 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:01.381 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:01.381 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:01.381 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:01.381 INFO:teuthology.orchestra.run.smithi159.stdout: ceph x86_64 2:18.2.4-843.g2c01458a.el9 ceph 6.5 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mds x86_64 2:18.2.4-843.g2c01458a.el9 ceph 2.1 M 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr x86_64 2:18.2.4-843.g2c01458a.el9 ceph 1.5 M 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-modules-core noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 246 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mon x86_64 2:18.2.4-843.g2c01458a.el9 ceph 4.7 M 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-osd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 17 M 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-13T14:12:01.382 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-13T14:12:01.383 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:01.384 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout:Install 33 Packages 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 29 M 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 104 M 2024-09-13T14:12:01.385 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:01.515 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repo Size 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-843.g2c01458a.el9 ceph 143 k 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:01.516 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:01.517 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 143 k 2024-09-13T14:12:01.517 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 395 k 2024-09-13T14:12:01.517 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:01.737 INFO:teuthology.orchestra.run.smithi042.stdout:ceph-immutable-object-cache-18.2.4-843.g2c01458 650 kB/s | 143 kB 00:00 2024-09-13T14:12:01.737 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:01.738 INFO:teuthology.orchestra.run.smithi042.stdout:Total 646 kB/s | 143 kB 00:00 2024-09-13T14:12:01.738 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:01.746 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:01.746 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:01.806 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:01.806 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:01.954 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:02.006 INFO:teuthology.orchestra.run.smithi159.stdout:(1/33): ceph-18.2.4-843.g2c01458a.el9.x86_64.rp 47 kB/s | 6.5 kB 00:00 2024-09-13T14:12:02.012 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:02.040 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:02.040 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:02.040 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:12:02.040 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:02.190 INFO:teuthology.orchestra.run.smithi159.stdout:(2/33): ceph-mgr-18.2.4-843.g2c01458a.el9.x86_6 4.5 MB/s | 1.5 MB 00:00 2024-09-13T14:12:02.256 INFO:teuthology.orchestra.run.smithi159.stdout:(3/33): ceph-mds-18.2.4-843.g2c01458a.el9.x86_6 5.4 MB/s | 2.1 MB 00:00 2024-09-13T14:12:02.373 INFO:teuthology.orchestra.run.smithi159.stdout:(4/33): ceph-mon-18.2.4-843.g2c01458a.el9.x86_6 13 MB/s | 4.7 MB 00:00 2024-09-13T14:12:02.399 INFO:teuthology.orchestra.run.smithi159.stdout:(5/33): ceph-mgr-modules-core-18.2.4-843.g2c014 1.7 MB/s | 246 kB 00:00 2024-09-13T14:12:02.549 INFO:teuthology.orchestra.run.smithi159.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.4 MB/s | 253 kB 00:00 2024-09-13T14:12:02.791 INFO:teuthology.orchestra.run.smithi159.stdout:(7/33): ceph-osd-18.2.4-843.g2c01458a.el9.x86_6 28 MB/s | 17 MB 00:00 2024-09-13T14:12:02.816 INFO:teuthology.orchestra.run.smithi159.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 399 kB/s | 106 kB 00:00 2024-09-13T14:12:02.850 INFO:teuthology.orchestra.run.smithi159.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 2.7 MB/s | 1.2 MB 00:00 2024-09-13T14:12:02.891 INFO:teuthology.orchestra.run.smithi159.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 1.6 MB/s | 125 kB 00:00 2024-09-13T14:12:02.942 INFO:teuthology.orchestra.run.smithi159.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 2.3 MB/s | 215 kB 00:00 2024-09-13T14:12:02.968 INFO:teuthology.orchestra.run.smithi159.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 768 kB/s | 135 kB 00:00 2024-09-13T14:12:02.974 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:02.974 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:02.974 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:02.974 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:02.974 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:02.975 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:03.017 INFO:teuthology.orchestra.run.smithi159.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.3 MB/s | 172 kB 00:00 2024-09-13T14:12:03.034 INFO:teuthology.orchestra.run.smithi159.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 377 kB/s | 35 kB 00:00 2024-09-13T14:12:03.051 INFO:teuthology.orchestra.run.smithi159.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 608 kB/s | 51 kB 00:00 2024-09-13T14:12:03.068 INFO:teuthology.orchestra.run.smithi159.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 824 kB/s | 42 kB 00:00 2024-09-13T14:12:03.111 INFO:teuthology.orchestra.run.smithi159.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 573 kB/s | 43 kB 00:00 2024-09-13T14:12:03.127 INFO:teuthology.orchestra.run.smithi159.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 726 kB/s | 11 kB 00:00 2024-09-13T14:12:03.153 INFO:teuthology.orchestra.run.smithi159.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.7 MB/s | 172 kB 00:00 2024-09-13T14:12:03.178 INFO:teuthology.orchestra.run.smithi159.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 352 kB/s | 18 kB 00:00 2024-09-13T14:12:03.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-mgr 2024-09-13T14:12:03.203 INFO:teuthology.orchestra.run.smithi159.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 460 kB/s | 23 kB 00:00 2024-09-13T14:12:03.228 INFO:teuthology.orchestra.run.smithi159.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-13T14:12:03.254 INFO:teuthology.orchestra.run.smithi159.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 393 kB/s | 20 kB 00:00 2024-09-13T14:12:03.287 INFO:teuthology.orchestra.run.smithi159.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.7 MB/s | 358 kB 00:00 2024-09-13T14:12:03.313 INFO:teuthology.orchestra.run.smithi159.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 551 kB/s | 46 kB 00:00 2024-09-13T14:12:03.346 INFO:teuthology.orchestra.run.smithi159.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 855 kB/s | 79 kB 00:00 2024-09-13T14:12:03.372 INFO:teuthology.orchestra.run.smithi159.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 281 kB/s | 16 kB 00:00 2024-09-13T14:12:03.397 INFO:teuthology.orchestra.run.smithi159.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2024-09-13T14:12:03.422 INFO:teuthology.orchestra.run.smithi159.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 712 kB/s | 36 kB 00:00 2024-09-13T14:12:03.448 INFO:teuthology.orchestra.run.smithi159.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.7 MB/s | 272 kB 00:00 2024-09-13T14:12:03.473 INFO:teuthology.orchestra.run.smithi159.stdout:(31/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-13T14:12:03.498 INFO:teuthology.orchestra.run.smithi159.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 399 kB/s | 20 kB 00:00 2024-09-13T14:12:03.523 INFO:teuthology.orchestra.run.smithi159.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.1 MB/s | 427 kB 00:00 2024-09-13T14:12:03.528 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:03.528 INFO:teuthology.orchestra.run.smithi159.stdout:Total 14 MB/s | 29 MB 00:02 2024-09-13T14:12:03.650 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:03.689 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:03.689 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:03.780 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:55 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:03.875 INFO:teuthology.orchestra.run.smithi042.stdout:Package ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:03.919 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:03.920 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:03.920 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:03.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:12:04.092 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:04.092 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:04.578 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:00:56 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:04.716 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-dashboard noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 3.5 M 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:12:04.717 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-grafana-dashboards noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 24 k 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-prometheus-alerts noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 16 k 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout:Install 5 Packages 2024-09-13T14:12:04.718 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:04.719 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 3.8 M 2024-09-13T14:12:04.719 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 87 M 2024-09-13T14:12:04.719 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:05.140 INFO:teuthology.orchestra.run.smithi042.stdout:(1/5): ceph-prometheus-alerts-18.2.4-843.g2c014 103 kB/s | 16 kB 00:00 2024-09-13T14:12:05.165 INFO:teuthology.orchestra.run.smithi042.stdout:(2/5): ceph-grafana-dashboards-18.2.4-843.g2c01 131 kB/s | 24 kB 00:00 2024-09-13T14:12:05.224 INFO:teuthology.orchestra.run.smithi042.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 367 kB/s | 31 kB 00:00 2024-09-13T14:12:05.258 INFO:teuthology.orchestra.run.smithi042.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.0 MB/s | 188 kB 00:00 2024-09-13T14:12:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:05.350 INFO:teuthology.orchestra.run.smithi042.stdout:(5/5): ceph-mgr-dashboard-18.2.4-843.g2c01458a. 9.6 MB/s | 3.5 MB 00:00 2024-09-13T14:12:05.354 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:05.354 INFO:teuthology.orchestra.run.smithi042.stdout:Total 6.0 MB/s | 3.8 MB 00:00 2024-09-13T14:12:05.354 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:05.359 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-13T14:12:05.362 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:05.362 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:05.391 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-13T14:12:05.424 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-13T14:12:05.454 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:05.455 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:05.517 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-13T14:12:05.611 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-13T14:12:05.652 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-13T14:12:05.704 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-13T14:12:05.715 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:05.742 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-13T14:12:05.785 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-13T14:12:05.800 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-13T14:12:05.818 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-13T14:12:05.830 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-13T14:12:05.859 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.no 3/5 2024-09-13T14:12:05.859 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-13T14:12:05.902 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-13T14:12:05.937 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-13T14:12:05.985 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-13T14:12:06.064 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-13T14:12:06.202 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-13T14:12:06.283 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-13T14:12:06.344 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-13T14:12:06.383 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-13T14:12:06.440 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-13T14:12:06.495 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-13T14:12:06.683 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-13T14:12:06.745 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-13T14:12:06.814 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-13T14:12:06.904 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-13T14:12:06.953 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-13T14:12:06.976 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.n 4/5 2024-09-13T14:12:07.012 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 5/5 2024-09-13T14:12:07.068 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-13T14:12:07.219 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.n 28/33 2024-09-13T14:12:07.247 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 29/33 2024-09-13T14:12:07.272 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 29/33 2024-09-13T14:12:07.272 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:07.272 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-13T14:12:07.273 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:12:07.273 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-13T14:12:07.273 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 5/5 2024-09-13T14:12:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.n 1/5 2024-09-13T14:12:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 2/5 2024-09-13T14:12:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.no 3/5 2024-09-13T14:12:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-13T14:12:08.355 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-13T14:12:08.355 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:08.355 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:08.356 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:08.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:12:08.740 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 30/33 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 30/33 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-13T14:12:08.769 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:09.156 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:01 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:09.225 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 31/33 2024-09-13T14:12:09.248 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 31/33 2024-09-13T14:12:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-13T14:12:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:12:09.249 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-13T14:12:09.249 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:09.297 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout:====================================================================================== 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout:====================================================================================== 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 7.4 M 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:12:09.298 INFO:teuthology.orchestra.run.smithi042.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout:====================================================================================== 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout:Install 12 Packages 2024-09-13T14:12:09.299 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:09.300 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 38 M 2024-09-13T14:12:09.300 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 212 M 2024-09-13T14:12:09.300 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:09.458 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 32/33 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 32/33 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-13T14:12:09.482 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:09.562 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-2:18.2.4-843.g2c01458a.el9.x86_64 33/33 2024-09-13T14:12:09.640 INFO:teuthology.orchestra.run.smithi042.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.1 MB/s | 191 kB 00:00 2024-09-13T14:12:09.723 INFO:teuthology.orchestra.run.smithi042.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 393 kB/s | 33 kB 00:00 2024-09-13T14:12:09.791 INFO:teuthology.orchestra.run.smithi042.stdout:(3/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-13T14:12:09.890 INFO:teuthology.orchestra.run.smithi042.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 184 kB/s | 18 kB 00:00 2024-09-13T14:12:09.991 INFO:teuthology.orchestra.run.smithi042.stdout:(5/12): ceph-mgr-diskprediction-local-18.2.4-84 14 MB/s | 7.4 MB 00:00 2024-09-13T14:12:10.008 INFO:teuthology.orchestra.run.smithi042.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 326 kB/s | 38 kB 00:00 2024-09-13T14:12:10.067 INFO:teuthology.orchestra.run.smithi042.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 8.7 MB/s | 3.0 MB 00:00 2024-09-13T14:12:10.134 INFO:teuthology.orchestra.run.smithi042.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.9 MB/s | 245 kB 00:00 2024-09-13T14:12:10.209 INFO:teuthology.orchestra.run.smithi042.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 36 MB/s | 5.1 MB 00:00 2024-09-13T14:12:10.234 INFO:teuthology.orchestra.run.smithi042.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 2.4 MB/s | 250 kB 00:00 2024-09-13T14:12:10.343 INFO:teuthology.orchestra.run.smithi042.stdout:(11/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 14 MB/s | 4.9 MB 00:00 2024-09-13T14:12:10.568 INFO:teuthology.orchestra.run.smithi042.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 45 MB/s | 16 MB 00:00 2024-09-13T14:12:10.569 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:10.569 INFO:teuthology.orchestra.run.smithi042.stdout:Total 30 MB/s | 38 MB 00:01 2024-09-13T14:12:10.936 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:10.963 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:10.963 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:11.285 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-2:18.2.4-843.g2c01458a.el9.x86_64 33/33 2024-09-13T14:12:11.285 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-2:18.2.4-843.g2c01458a.el9.x86_64 1/33 2024-09-13T14:12:11.285 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 2/33 2024-09-13T14:12:11.285 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 3/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 4/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 5/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.n 6/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-13T14:12:11.286 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-13T14:12:11.287 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-13T14:12:11.288 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-13T14:12:11.288 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-13T14:12:11.288 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-13T14:12:11.288 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-13T14:12:11.288 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-13T14:12:11.289 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-13T14:12:11.289 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-13T14:12:11.487 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:11.487 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:12.161 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-13T14:12:12.161 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:12.161 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:12.161 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:12.161 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mds-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-modules-core-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mon-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-osd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-13T14:12:12.162 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-ply-3.11-14.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-13T14:12:12.163 INFO:teuthology.orchestra.run.smithi159.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-13T14:12:12.164 INFO:teuthology.orchestra.run.smithi159.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-13T14:12:12.164 INFO:teuthology.orchestra.run.smithi159.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-13T14:12:12.164 INFO:teuthology.orchestra.run.smithi159.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-13T14:12:12.164 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:12.164 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:12.276 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:12.351 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:12:12.392 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:12:12.497 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:12:12.534 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:12:12.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-base 2024-09-13T14:12:12.727 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:12:12.758 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:12:12.837 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:12:13.146 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:49 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:13.238 INFO:teuthology.orchestra.run.smithi159.stdout:Package ceph-base-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:13.280 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:13.281 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:13.281 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:13.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install cephadm 2024-09-13T14:12:13.540 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:12:13.659 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:12:13.915 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:49 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:14.050 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:14.050 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:14.050 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout: cephadm noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 226 k 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 226 k 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 217 k 2024-09-13T14:12:14.051 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:14.285 INFO:teuthology.orchestra.run.smithi159.stdout:cephadm-18.2.4-843.g2c01458a.el9.noarch.rpm 967 kB/s | 226 kB 00:00 2024-09-13T14:12:14.286 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:14.286 INFO:teuthology.orchestra.run.smithi159.stdout:Total 961 kB/s | 226 kB 00:00 2024-09-13T14:12:14.286 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:14.291 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:14.291 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:14.294 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:14.294 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:14.367 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:14.824 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:14.914 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:14.960 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:12:15.270 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/1 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout: cephadm-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:15.529 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:15.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-immutable-object-cache 2024-09-13T14:12:16.036 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:12:16.065 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 12/12 2024-09-13T14:12:16.345 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:52 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:16.478 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:16.479 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:16.479 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repo Size 2024-09-13T14:12:16.479 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:16.479 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-843.g2c01458a.el9 ceph 143 k 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 143 k 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 395 k 2024-09-13T14:12:16.480 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:16.684 INFO:teuthology.orchestra.run.smithi159.stdout:ceph-immutable-object-cache-18.2.4-843.g2c01458 701 kB/s | 143 kB 00:00 2024-09-13T14:12:16.684 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:16.685 INFO:teuthology.orchestra.run.smithi159.stdout:Total 695 kB/s | 143 kB 00:00 2024-09-13T14:12:16.685 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:16.692 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:16.693 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:16.750 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:16.750 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:16.897 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:16.993 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:17.018 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:17.018 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:17.018 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-13T14:12:17.018 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.e 1/1 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-immutable-object-cache-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:17.920 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:18.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-mgr 2024-09-13T14:12:18.559 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 12/12 2024-09-13T14:12:18.559 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 1/12 2024-09-13T14:12:18.559 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:12:18.559 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:12:18.559 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:12:18.560 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:12:18.719 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:54 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:18.812 INFO:teuthology.orchestra.run.smithi159.stdout:Package ceph-mgr-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:18.856 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:18.857 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:18.857 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:18.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-mgr-dashboard 2024-09-13T14:12:18.966 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:12:18.966 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:18.966 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-diskprediction-local-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:18.967 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:19.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:12:19.493 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:00:55 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:19.625 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-dashboard noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 3.5 M 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-grafana-dashboards noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 24 k 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-prometheus-alerts noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 16 k 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout:Install 5 Packages 2024-09-13T14:12:19.626 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:19.627 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 3.8 M 2024-09-13T14:12:19.627 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 87 M 2024-09-13T14:12:19.627 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:19.923 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:11 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:20.063 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-rook noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 49 k 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:12:20.064 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout:Install 13 Packages 2024-09-13T14:12:20.065 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:20.066 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 2.2 M 2024-09-13T14:12:20.066 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 26 M 2024-09-13T14:12:20.066 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:20.180 INFO:teuthology.orchestra.run.smithi159.stdout:(1/5): ceph-prometheus-alerts-18.2.4-843.g2c014 104 kB/s | 16 kB 00:00 2024-09-13T14:12:20.206 INFO:teuthology.orchestra.run.smithi159.stdout:(2/5): ceph-grafana-dashboards-18.2.4-843.g2c01 133 kB/s | 24 kB 00:00 2024-09-13T14:12:20.256 INFO:teuthology.orchestra.run.smithi159.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 409 kB/s | 31 kB 00:00 2024-09-13T14:12:20.289 INFO:teuthology.orchestra.run.smithi159.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.2 MB/s | 188 kB 00:00 2024-09-13T14:12:20.340 INFO:teuthology.orchestra.run.smithi042.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 254 kB/s | 26 kB 00:00 2024-09-13T14:12:20.382 INFO:teuthology.orchestra.run.smithi042.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 134 kB/s | 19 kB 00:00 2024-09-13T14:12:20.406 INFO:teuthology.orchestra.run.smithi159.stdout:(5/5): ceph-mgr-dashboard-18.2.4-843.g2c01458a. 9.2 MB/s | 3.5 MB 00:00 2024-09-13T14:12:20.410 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:20.410 INFO:teuthology.orchestra.run.smithi159.stdout:Total 4.8 MB/s | 3.8 MB 00:00 2024-09-13T14:12:20.410 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:20.417 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:20.417 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:20.424 INFO:teuthology.orchestra.run.smithi042.stdout:(3/13): ceph-mgr-rook-18.2.4-843.g2c01458a.el9. 266 kB/s | 49 kB 00:00 2024-09-13T14:12:20.449 INFO:teuthology.orchestra.run.smithi042.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.0 MB/s | 222 kB 00:00 2024-09-13T14:12:20.491 INFO:teuthology.orchestra.run.smithi042.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.4 MB/s | 159 kB 00:00 2024-09-13T14:12:20.506 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:20.507 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:20.508 INFO:teuthology.orchestra.run.smithi042.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 918 kB/s | 54 kB 00:00 2024-09-13T14:12:20.550 INFO:teuthology.orchestra.run.smithi042.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 549 kB/s | 32 kB 00:00 2024-09-13T14:12:20.567 INFO:teuthology.orchestra.run.smithi042.stdout:(8/13): python3-certifi-2023.05.07-4.el9.noarch 241 kB/s | 14 kB 00:00 2024-09-13T14:12:20.592 INFO:teuthology.orchestra.run.smithi042.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.6 MB/s | 279 kB 00:00 2024-09-13T14:12:20.626 INFO:teuthology.orchestra.run.smithi042.stdout:(10/13): python3-google-auth-2.34.0-1.el9.noarc 2.8 MB/s | 220 kB 00:00 2024-09-13T14:12:20.659 INFO:teuthology.orchestra.run.smithi042.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 2.8 MB/s | 90 kB 00:00 2024-09-13T14:12:20.676 INFO:teuthology.orchestra.run.smithi042.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 708 kB/s | 59 kB 00:00 2024-09-13T14:12:20.710 INFO:teuthology.orchestra.run.smithi042.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2024-09-13T14:12:20.713 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:20.713 INFO:teuthology.orchestra.run.smithi042.stdout:Total 3.4 MB/s | 2.2 MB 00:00 2024-09-13T14:12:20.758 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:20.780 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:20.796 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:20.796 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:20.855 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-13T14:12:20.886 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-13T14:12:20.894 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:20.895 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:20.915 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.no 3/5 2024-09-13T14:12:21.538 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:21.714 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:12:21.772 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:12:21.826 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:12:21.862 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:12:21.904 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:12:21.961 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.n 4/5 2024-09-13T14:12:22.002 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 5/5 2024-09-13T14:12:22.018 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:12:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:12:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:12:22.947 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:12:23.002 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:12:23.036 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:12:23.085 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:12:23.113 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 13/13 2024-09-13T14:12:23.120 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 5/5 2024-09-13T14:12:23.120 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.n 1/5 2024-09-13T14:12:23.120 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 2/5 2024-09-13T14:12:23.120 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.no 3/5 2024-09-13T14:12:23.120 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-13T14:12:23.394 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-13T14:12:23.394 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:23.394 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:23.394 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-grafana-dashboards-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-dashboard-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-prometheus-alerts-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:23.395 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:23.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 13/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 1/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:12:23.864 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:12:23.865 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:12:23.865 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:12:24.192 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:00 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:24.205 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:12:24.206 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:24.207 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:24.325 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:24.326 INFO:teuthology.orchestra.run.smithi159.stdout:====================================================================================== 2024-09-13T14:12:24.326 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout:====================================================================================== 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 7.4 M 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-13T14:12:24.327 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout:====================================================================================== 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout:Install 12 Packages 2024-09-13T14:12:24.328 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:24.329 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 38 M 2024-09-13T14:12:24.329 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 212 M 2024-09-13T14:12:24.329 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:24.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:12:24.964 INFO:teuthology.orchestra.run.smithi159.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 797 kB/s | 191 kB 00:00 2024-09-13T14:12:25.039 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:17 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:25.056 INFO:teuthology.orchestra.run.smithi159.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.4 MB/s | 801 kB 00:00 2024-09-13T14:12:25.081 INFO:teuthology.orchestra.run.smithi159.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 279 kB/s | 33 kB 00:00 2024-09-13T14:12:25.177 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-cephadm noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 139 k 2024-09-13T14:12:25.178 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout:Install 6 Packages 2024-09-13T14:12:25.179 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:25.180 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 7.0 M 2024-09-13T14:12:25.180 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 33 M 2024-09-13T14:12:25.180 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:25.214 INFO:teuthology.orchestra.run.smithi159.stdout:(4/12): ceph-mgr-diskprediction-local-18.2.4-84 15 MB/s | 7.4 MB 00:00 2024-09-13T14:12:25.240 INFO:teuthology.orchestra.run.smithi159.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 115 kB/s | 18 kB 00:00 2024-09-13T14:12:25.274 INFO:teuthology.orchestra.run.smithi159.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 657 kB/s | 38 kB 00:00 2024-09-13T14:12:25.357 INFO:teuthology.orchestra.run.smithi159.stdout:(7/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 9.9 MB/s | 3.0 MB 00:00 2024-09-13T14:12:25.399 INFO:teuthology.orchestra.run.smithi159.stdout:(8/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.9 MB/s | 245 kB 00:00 2024-09-13T14:12:25.508 INFO:teuthology.orchestra.run.smithi159.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 34 MB/s | 5.1 MB 00:00 2024-09-13T14:12:25.533 INFO:teuthology.orchestra.run.smithi159.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.8 MB/s | 250 kB 00:00 2024-09-13T14:12:25.675 INFO:teuthology.orchestra.run.smithi159.stdout:(11/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 11 MB/s | 4.9 MB 00:00 2024-09-13T14:12:25.753 INFO:teuthology.orchestra.run.smithi042.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.8 MB/s | 248 kB 00:00 2024-09-13T14:12:25.828 INFO:teuthology.orchestra.run.smithi042.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 7.2 MB/s | 548 kB 00:00 2024-09-13T14:12:25.850 INFO:teuthology.orchestra.run.smithi159.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 48 MB/s | 16 MB 00:00 2024-09-13T14:12:25.851 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:25.851 INFO:teuthology.orchestra.run.smithi159.stdout:Total 25 MB/s | 38 MB 00:01 2024-09-13T14:12:25.853 INFO:teuthology.orchestra.run.smithi042.stdout:(3/6): ceph-mgr-cephadm-18.2.4-843.g2c01458a.el 584 kB/s | 139 kB 00:00 2024-09-13T14:12:25.904 INFO:teuthology.orchestra.run.smithi042.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 765 kB/s | 58 kB 00:00 2024-09-13T14:12:25.946 INFO:teuthology.orchestra.run.smithi042.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 810 kB/s | 74 kB 00:00 2024-09-13T14:12:26.206 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:26.231 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:26.232 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:26.756 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:26.756 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:27.188 INFO:teuthology.orchestra.run.smithi042.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.8 MB/s | 6.0 MB 00:01 2024-09-13T14:12:27.191 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:27.191 INFO:teuthology.orchestra.run.smithi042.stdout:Total 3.5 MB/s | 7.0 MB 00:02 2024-09-13T14:12:27.267 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:27.279 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:27.280 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:27.386 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:27.387 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:27.514 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:27.598 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-13T14:12:27.640 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:12:27.704 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:27.747 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:12:27.799 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-13T14:12:27.876 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:12:27.940 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:12:27.984 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-13T14:12:28.015 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-13T14:12:28.092 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-13T14:12:28.492 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:12:28.579 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:12:28.648 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:12:28.687 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 6/6 2024-09-13T14:12:28.746 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-13T14:12:28.861 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 6/6 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/6 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:12:29.446 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:12:29.721 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:29.722 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:30.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install ceph-fuse 2024-09-13T14:12:30.116 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:12:30.618 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:22 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:30.757 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:30.757 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repository Size 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-fuse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 852 k 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:30.758 INFO:teuthology.orchestra.run.smithi042.stdout:Install 2 Packages 2024-09-13T14:12:30.759 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:30.759 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 932 k 2024-09-13T14:12:30.759 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 2.7 M 2024-09-13T14:12:30.759 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:30.989 INFO:teuthology.orchestra.run.smithi042.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 901 kB/s | 80 kB 00:00 2024-09-13T14:12:31.148 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-13T14:12:31.173 INFO:teuthology.orchestra.run.smithi042.stdout:(2/2): ceph-fuse-18.2.4-843.g2c01458a.el9.x86_6 3.1 MB/s | 852 kB 00:00 2024-09-13T14:12:31.173 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:31.173 INFO:teuthology.orchestra.run.smithi042.stdout:Total 2.2 MB/s | 932 kB 00:00 2024-09-13T14:12:31.181 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 12/12 2024-09-13T14:12:31.185 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:31.208 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:31.208 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:31.293 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:31.294 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:31.486 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:31.625 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:12:31.709 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2/2 2024-09-13T14:12:32.336 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2/2 2024-09-13T14:12:32.336 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/2 2024-09-13T14:12:32.609 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:12:32.609 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:32.609 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:32.609 INFO:teuthology.orchestra.run.smithi042.stdout: ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:12:32.610 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:32.610 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:32.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install librados-devel 2024-09-13T14:12:33.461 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:25 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:33.600 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repo Size 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout: librados-devel x86_64 2:18.2.4-843.g2c01458a.el9 ceph 127 k 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:33.601 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:33.602 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 127 k 2024-09-13T14:12:33.602 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 456 k 2024-09-13T14:12:33.602 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:33.818 INFO:teuthology.orchestra.run.smithi042.stdout:librados-devel-18.2.4-843.g2c01458a.el9.x86_64. 585 kB/s | 127 kB 00:00 2024-09-13T14:12:33.819 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:33.819 INFO:teuthology.orchestra.run.smithi042.stdout:Total 581 kB/s | 127 kB 00:00 2024-09-13T14:12:33.819 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:33.825 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:33.825 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 12/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-843.g2c0145 1/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-13T14:12:33.850 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-13T14:12:33.851 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-13T14:12:33.851 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-13T14:12:33.851 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-13T14:12:33.893 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:33.893 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:34.076 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:34.177 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:34.278 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-13T14:12:34.278 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:34.278 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:34.278 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-diskprediction-local-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:34.278 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:34.279 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:34.644 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-mgr-rook 2024-09-13T14:12:34.868 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:34.868 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:34.868 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:34.869 INFO:teuthology.orchestra.run.smithi042.stdout: librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:34.869 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:34.869 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:35.112 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install libcephfs2 2024-09-13T14:12:35.222 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:11 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:35.357 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:35.358 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-rook noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 49 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-13T14:12:35.359 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout:Install 13 Packages 2024-09-13T14:12:35.360 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:35.361 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 2.2 M 2024-09-13T14:12:35.361 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 26 M 2024-09-13T14:12:35.361 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:35.565 INFO:teuthology.orchestra.run.smithi159.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 342 kB/s | 26 kB 00:00 2024-09-13T14:12:35.582 INFO:teuthology.orchestra.run.smithi159.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 209 kB/s | 19 kB 00:00 2024-09-13T14:12:35.661 INFO:teuthology.orchestra.run.smithi159.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.9 MB/s | 222 kB 00:00 2024-09-13T14:12:35.683 INFO:teuthology.orchestra.run.smithi159.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.6 MB/s | 159 kB 00:00 2024-09-13T14:12:35.700 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:27 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:35.708 INFO:teuthology.orchestra.run.smithi159.stdout:(5/13): ceph-mgr-rook-18.2.4-843.g2c01458a.el9. 226 kB/s | 49 kB 00:00 2024-09-13T14:12:35.734 INFO:teuthology.orchestra.run.smithi159.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-09-13T14:12:35.759 INFO:teuthology.orchestra.run.smithi159.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 2.7 MB/s | 279 kB 00:00 2024-09-13T14:12:35.795 INFO:teuthology.orchestra.run.smithi042.stdout:Package libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:35.809 INFO:teuthology.orchestra.run.smithi159.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 320 kB/s | 32 kB 00:00 2024-09-13T14:12:35.834 INFO:teuthology.orchestra.run.smithi159.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 140 kB/s | 14 kB 00:00 2024-09-13T14:12:35.840 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:35.841 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:35.841 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:35.868 INFO:teuthology.orchestra.run.smithi159.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 1.7 MB/s | 59 kB 00:00 2024-09-13T14:12:35.893 INFO:teuthology.orchestra.run.smithi159.stdout:(11/13): python3-google-auth-2.34.0-1.el9.noarc 1.8 MB/s | 220 kB 00:00 2024-09-13T14:12:35.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install libcephfs-devel 2024-09-13T14:12:35.918 INFO:teuthology.orchestra.run.smithi159.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-09-13T14:12:35.960 INFO:teuthology.orchestra.run.smithi159.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.8 MB/s | 1.0 MB 00:00 2024-09-13T14:12:35.963 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:35.963 INFO:teuthology.orchestra.run.smithi159.stdout:Total 3.7 MB/s | 2.2 MB 00:00 2024-09-13T14:12:36.027 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:36.042 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:36.042 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:36.130 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:36.131 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:36.499 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:28 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:36.633 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:36.636 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repo Size 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout: libcephfs-devel x86_64 2:18.2.4-843.g2c01458a.el9 ceph 31 k 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:36.637 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:36.638 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:36.638 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:36.638 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 31 k 2024-09-13T14:12:36.638 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 139 k 2024-09-13T14:12:36.639 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:36.796 INFO:teuthology.orchestra.run.smithi042.stdout:libcephfs-devel-18.2.4-843.g2c01458a.el9.x86_64 196 kB/s | 31 kB 00:00 2024-09-13T14:12:36.797 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:36.797 INFO:teuthology.orchestra.run.smithi042.stdout:Total 195 kB/s | 31 kB 00:00 2024-09-13T14:12:36.797 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:36.801 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:36.801 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:36.822 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:36.822 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:36.825 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-13T14:12:36.895 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-13T14:12:36.903 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:36.959 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-13T14:12:37.004 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:37.005 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-13T14:12:37.046 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-13T14:12:37.148 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-13T14:12:37.259 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-13T14:12:37.323 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-13T14:12:37.392 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout: libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:37.636 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:37.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install librados2 2024-09-13T14:12:38.038 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-13T14:12:38.095 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-13T14:12:38.129 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-13T14:12:38.178 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-13T14:12:38.215 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 13/13 2024-09-13T14:12:38.399 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:30 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:38.492 INFO:teuthology.orchestra.run.smithi042.stdout:Package librados2-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:38.539 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:38.540 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:38.540 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:38.611 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install librbd1 2024-09-13T14:12:39.018 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 13/13 2024-09-13T14:12:39.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 1/13 2024-09-13T14:12:39.018 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-13T14:12:39.019 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-13T14:12:39.199 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:31 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:39.293 INFO:teuthology.orchestra.run.smithi042.stdout:Package librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:39.338 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:39.339 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:39.339 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-rook-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-13T14:12:39.360 INFO:teuthology.orchestra.run.smithi159.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:39.361 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:39.410 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install python3-rados 2024-09-13T14:12:39.620 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-mgr-cephadm 2024-09-13T14:12:40.001 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:40.096 INFO:teuthology.orchestra.run.smithi042.stdout:Package python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:40.141 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:40.142 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:40.143 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:40.190 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:16 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:40.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install python3-rgw 2024-09-13T14:12:40.323 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-cephadm noarch 2:18.2.4-843.g2c01458a.el9 ceph-noarch 139 k 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-13T14:12:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout:Install 6 Packages 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:40.325 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 7.0 M 2024-09-13T14:12:40.326 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 33 M 2024-09-13T14:12:40.326 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:40.794 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:40.888 INFO:teuthology.orchestra.run.smithi042.stdout:Package python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:40.935 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:40.936 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:40.936 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:40.980 INFO:teuthology.orchestra.run.smithi159.stdout:(1/6): ceph-mgr-cephadm-18.2.4-843.g2c01458a.el 693 kB/s | 139 kB 00:00 2024-09-13T14:12:41.014 INFO:teuthology.orchestra.run.smithi159.stdout:(2/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.0 MB/s | 248 kB 00:00 2024-09-13T14:12:41.055 INFO:teuthology.orchestra.run.smithi159.stdout:(3/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 7.1 MB/s | 548 kB 00:00 2024-09-13T14:12:41.081 INFO:teuthology.orchestra.run.smithi159.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 3.1 MB/s | 74 kB 00:00 2024-09-13T14:12:41.106 INFO:teuthology.orchestra.run.smithi159.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 639 kB/s | 58 kB 00:00 2024-09-13T14:12:41.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install python3-cephfs 2024-09-13T14:12:41.733 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:33 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:41.827 INFO:teuthology.orchestra.run.smithi042.stdout:Package python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:41.872 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:41.873 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:41.873 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:41.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install python3-rbd 2024-09-13T14:12:42.539 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:42.634 INFO:teuthology.orchestra.run.smithi042.stdout:Package python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:42.679 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:42.680 INFO:teuthology.orchestra.run.smithi042.stdout:Nothing to do. 2024-09-13T14:12:42.680 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:42.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install rbd-fuse 2024-09-13T14:12:42.798 INFO:teuthology.orchestra.run.smithi159.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 3.0 MB/s | 6.0 MB 00:02 2024-09-13T14:12:42.800 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:42.801 INFO:teuthology.orchestra.run.smithi159.stdout:Total 2.8 MB/s | 7.0 MB 00:02 2024-09-13T14:12:42.875 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:42.886 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:42.886 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:42.986 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:42.987 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:43.301 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:43.349 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:35 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:43.481 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-13T14:12:43.488 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout: Package Architecture Version Repository Size 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-fuse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 87 k 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:43.489 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:43.490 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 87 k 2024-09-13T14:12:43.490 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 227 k 2024-09-13T14:12:43.490 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:43.525 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-13T14:12:43.684 INFO:teuthology.orchestra.run.smithi042.stdout:rbd-fuse-18.2.4-843.g2c01458a.el9.x86_64.rpm 448 kB/s | 87 kB 00:00 2024-09-13T14:12:43.684 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:43.684 INFO:teuthology.orchestra.run.smithi042.stdout:Total 445 kB/s | 87 kB 00:00 2024-09-13T14:12:43.685 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:43.692 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:43.692 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:43.758 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:43.759 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:43.910 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:44.016 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:44.047 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-13T14:12:44.152 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-13T14:12:44.215 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-13T14:12:44.245 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 6/6 2024-09-13T14:12:44.369 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:44.610 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:44.611 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:44.611 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:44.611 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:44.611 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:44.611 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:44.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install rbd-mirror 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 6/6 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 1/6 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-13T14:12:45.062 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-13T14:12:45.324 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-13T14:12:45.324 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-mgr-cephadm-2:18.2.4-843.g2c01458a.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:45.325 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:45.448 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:37 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:45.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install ceph-fuse 2024-09-13T14:12:45.587 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:45.587 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:45.587 INFO:teuthology.orchestra.run.smithi042.stdout: Package Arch Version Repo Size 2024-09-13T14:12:45.587 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:45.587 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-mirror x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.0 M 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:45.588 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 3.0 M 2024-09-13T14:12:45.589 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 12 M 2024-09-13T14:12:45.589 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:45.928 INFO:teuthology.orchestra.run.smithi042.stdout:rbd-mirror-18.2.4-843.g2c01458a.el9.x86_64.rpm 8.8 MB/s | 3.0 MB 00:00 2024-09-13T14:12:45.928 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:45.929 INFO:teuthology.orchestra.run.smithi042.stdout:Total 8.8 MB/s | 3.0 MB 00:00 2024-09-13T14:12:45.929 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:45.939 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:45.939 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:46.031 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:46.031 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:46.145 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:22 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:46.280 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:46.280 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repository Size 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-fuse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 852 k 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:Installing dependencies: 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout:Install 2 Packages 2024-09-13T14:12:46.281 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:46.282 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 932 k 2024-09-13T14:12:46.282 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 2.7 M 2024-09-13T14:12:46.282 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:46.400 INFO:teuthology.orchestra.run.smithi159.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 849 kB/s | 80 kB 00:00 2024-09-13T14:12:46.482 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:46.545 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:12:46.571 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:46.600 INFO:teuthology.orchestra.run.smithi159.stdout:(2/2): ceph-fuse-18.2.4-843.g2c01458a.el9.x86_6 2.8 MB/s | 852 kB 00:00 2024-09-13T14:12:46.601 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:46.601 INFO:teuthology.orchestra.run.smithi159.stdout:Total 2.8 MB/s | 932 kB 00:00 2024-09-13T14:12:46.612 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:46.635 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:46.635 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:46.716 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:46.716 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:46.911 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:47.040 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-13T14:12:47.116 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2/2 2024-09-13T14:12:47.671 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:47.671 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:47.671 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:47.671 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:47.671 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:47.672 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:47.795 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2/2 2024-09-13T14:12:47.795 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/2 2024-09-13T14:12:47.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum -y install rbd-nbd 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout: ceph-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:48.084 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:48.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install librados-devel 2024-09-13T14:12:48.530 INFO:teuthology.orchestra.run.smithi042.stdout:Last metadata expiration check: 0:01:40 ago on Fri 13 Sep 2024 02:11:08 PM UTC. 2024-09-13T14:12:48.668 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout: Package Architecture Version Repository Size 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-nbd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 172 k 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:48.669 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout:Install 1 Package 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 172 k 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 494 k 2024-09-13T14:12:48.670 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T14:12:48.890 INFO:teuthology.orchestra.run.smithi042.stdout:rbd-nbd-18.2.4-843.g2c01458a.el9.x86_64.rpm 784 kB/s | 172 kB 00:00 2024-09-13T14:12:48.890 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:48.890 INFO:teuthology.orchestra.run.smithi042.stdout:Total 779 kB/s | 172 kB 00:00 2024-09-13T14:12:48.891 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T14:12:48.899 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T14:12:48.899 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T14:12:48.962 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:24 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:48.967 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T14:12:48.967 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T14:12:49.097 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:49.097 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:49.097 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repo Size 2024-09-13T14:12:49.097 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout: librados-devel x86_64 2:18.2.4-843.g2c01458a.el9 ceph 127 k 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:12:49.098 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:49.099 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 127 k 2024-09-13T14:12:49.099 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 456 k 2024-09-13T14:12:49.099 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:49.135 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T14:12:49.240 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:49.279 INFO:teuthology.orchestra.run.smithi159.stdout:librados-devel-18.2.4-843.g2c01458a.el9.x86_64. 701 kB/s | 127 kB 00:00 2024-09-13T14:12:49.280 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:49.281 INFO:teuthology.orchestra.run.smithi159.stdout:Total 694 kB/s | 127 kB 00:00 2024-09-13T14:12:49.281 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:49.286 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:49.286 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:49.349 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:49.350 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:49.502 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:49.602 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:49.651 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:49.943 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T14:12:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:12:49.943 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T14:12:50.044 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:50.206 DEBUG:teuthology.parallel:result is None 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout: librados-devel-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:50.301 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:50.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install libcephfs2 2024-09-13T14:12:51.093 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:27 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:51.186 INFO:teuthology.orchestra.run.smithi159.stdout:Package libcephfs2-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:51.230 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:51.231 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:51.231 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:51.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install libcephfs-devel 2024-09-13T14:12:51.926 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:27 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:52.059 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:52.060 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:52.060 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repo Size 2024-09-13T14:12:52.060 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:52.060 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs-devel x86_64 2:18.2.4-843.g2c01458a.el9 ceph 31 k 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 31 k 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 139 k 2024-09-13T14:12:52.061 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:52.209 INFO:teuthology.orchestra.run.smithi159.stdout:libcephfs-devel-18.2.4-843.g2c01458a.el9.x86_64 210 kB/s | 31 kB 00:00 2024-09-13T14:12:52.209 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:52.210 INFO:teuthology.orchestra.run.smithi159.stdout:Total 208 kB/s | 31 kB 00:00 2024-09-13T14:12:52.210 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:52.213 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:52.213 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:52.233 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:52.234 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:52.324 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:52.448 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:52.805 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:53.038 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:53.039 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:53.039 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:53.039 INFO:teuthology.orchestra.run.smithi159.stdout: libcephfs-devel-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:53.039 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:53.039 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:53.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install librados2 2024-09-13T14:12:53.797 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:29 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:53.887 INFO:teuthology.orchestra.run.smithi159.stdout:Package librados2-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:53.929 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:53.930 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:53.930 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:54.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install librbd1 2024-09-13T14:12:54.573 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:30 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:54.664 INFO:teuthology.orchestra.run.smithi159.stdout:Package librbd1-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:54.707 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:54.708 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:54.708 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:54.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install python3-rados 2024-09-13T14:12:55.337 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:31 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:55.427 INFO:teuthology.orchestra.run.smithi159.stdout:Package python3-rados-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:55.470 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:55.471 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:55.471 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:55.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install python3-rgw 2024-09-13T14:12:56.105 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:56.196 INFO:teuthology.orchestra.run.smithi159.stdout:Package python3-rgw-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:56.239 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:56.240 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:56.240 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:56.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install python3-cephfs 2024-09-13T14:12:56.871 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:32 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:56.962 INFO:teuthology.orchestra.run.smithi159.stdout:Package python3-cephfs-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:57.005 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:57.006 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:57.006 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:57.077 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install python3-rbd 2024-09-13T14:12:57.642 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:33 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:57.735 INFO:teuthology.orchestra.run.smithi159.stdout:Package python3-rbd-2:18.2.4-843.g2c01458a.el9.x86_64 is already installed. 2024-09-13T14:12:57.777 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:57.778 INFO:teuthology.orchestra.run.smithi159.stdout:Nothing to do. 2024-09-13T14:12:57.778 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:12:57.852 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install rbd-fuse 2024-09-13T14:12:58.423 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:34 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:12:58.557 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:12:58.557 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:58.557 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2024-09-13T14:12:58.557 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-fuse x86_64 2:18.2.4-843.g2c01458a.el9 ceph 87 k 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 87 k 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 227 k 2024-09-13T14:12:58.558 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:12:58.746 INFO:teuthology.orchestra.run.smithi159.stdout:rbd-fuse-18.2.4-843.g2c01458a.el9.x86_64.rpm 462 kB/s | 87 kB 00:00 2024-09-13T14:12:58.746 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:12:58.746 INFO:teuthology.orchestra.run.smithi159.stdout:Total 459 kB/s | 87 kB 00:00 2024-09-13T14:12:58.747 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:12:58.753 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:12:58.754 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:12:58.818 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:12:58.818 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:12:58.962 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:12:59.060 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:59.439 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:59.697 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:12:59.697 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:59.697 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:12:59.697 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-fuse-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:12:59.698 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:12:59.698 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:13:00.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install rbd-mirror 2024-09-13T14:13:00.584 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:36 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:13:00.722 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout: Package Arch Version Repo Size 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-mirror x86_64 2:18.2.4-843.g2c01458a.el9 ceph 3.0 M 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 3.0 M 2024-09-13T14:13:00.723 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 12 M 2024-09-13T14:13:00.724 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:13:01.058 INFO:teuthology.orchestra.run.smithi159.stdout:rbd-mirror-18.2.4-843.g2c01458a.el9.x86_64.rpm 8.9 MB/s | 3.0 MB 00:00 2024-09-13T14:13:01.059 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:13:01.059 INFO:teuthology.orchestra.run.smithi159.stdout:Total 8.9 MB/s | 3.0 MB 00:00 2024-09-13T14:13:01.059 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:13:01.068 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:13:01.068 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:13:01.154 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:13:01.154 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:13:01.577 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:13:01.751 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:01.777 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:01.777 INFO:teuthology.orchestra.run.smithi159.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-13T14:13:01.777 INFO:teuthology.orchestra.run.smithi159.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-13T14:13:01.778 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:13:01.778 INFO:teuthology.orchestra.run.smithi159.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-13T14:13:01.778 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-mirror-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:02.776 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:13:03.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo yum -y install rbd-nbd 2024-09-13T14:13:03.602 INFO:teuthology.orchestra.run.smithi159.stdout:Last metadata expiration check: 0:01:39 ago on Fri 13 Sep 2024 02:11:24 PM UTC. 2024-09-13T14:13:03.739 INFO:teuthology.orchestra.run.smithi159.stdout:Dependencies resolved. 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout: Package Architecture Version Repository Size 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout:Installing: 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-nbd x86_64 2:18.2.4-843.g2c01458a.el9 ceph 172 k 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction Summary 2024-09-13T14:13:03.740 INFO:teuthology.orchestra.run.smithi159.stdout:================================================================================ 2024-09-13T14:13:03.741 INFO:teuthology.orchestra.run.smithi159.stdout:Install 1 Package 2024-09-13T14:13:03.741 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:03.741 INFO:teuthology.orchestra.run.smithi159.stdout:Total download size: 172 k 2024-09-13T14:13:03.741 INFO:teuthology.orchestra.run.smithi159.stdout:Installed size: 494 k 2024-09-13T14:13:03.741 INFO:teuthology.orchestra.run.smithi159.stdout:Downloading Packages: 2024-09-13T14:13:03.953 INFO:teuthology.orchestra.run.smithi159.stdout:rbd-nbd-18.2.4-843.g2c01458a.el9.x86_64.rpm 809 kB/s | 172 kB 00:00 2024-09-13T14:13:03.954 INFO:teuthology.orchestra.run.smithi159.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:13:03.954 INFO:teuthology.orchestra.run.smithi159.stdout:Total 803 kB/s | 172 kB 00:00 2024-09-13T14:13:03.955 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction check 2024-09-13T14:13:03.962 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction check succeeded. 2024-09-13T14:13:03.962 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction test 2024-09-13T14:13:04.026 INFO:teuthology.orchestra.run.smithi159.stdout:Transaction test succeeded. 2024-09-13T14:13:04.026 INFO:teuthology.orchestra.run.smithi159.stdout:Running transaction 2024-09-13T14:13:04.190 INFO:teuthology.orchestra.run.smithi159.stdout: Preparing : 1/1 2024-09-13T14:13:04.300 INFO:teuthology.orchestra.run.smithi159.stdout: Installing : rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:04.715 INFO:teuthology.orchestra.run.smithi159.stdout: Running scriptlet: rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout: Verifying : rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 1/1 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout:Installed: 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout: rbd-nbd-2:18.2.4-843.g2c01458a.el9.x86_64 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:13:04.973 INFO:teuthology.orchestra.run.smithi159.stdout:Complete! 2024-09-13T14:13:05.247 DEBUG:teuthology.parallel:result is None 2024-09-13T14:13:05.247 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:13:05.248 INFO:teuthology.packaging:ref: None 2024-09-13T14:13:05.248 INFO:teuthology.packaging:tag: None 2024-09-13T14:13:05.248 INFO:teuthology.packaging:branch: reef 2024-09-13T14:13:05.248 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:13:05.248 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-13T14:13:05.521 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:13:05.548 INFO:teuthology.orchestra.run.smithi042.stdout:18.2.4-843.g2c01458a.el9 2024-09-13T14:13:05.548 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-843.g2c01458a.el9 2024-09-13T14:13:05.549 INFO:teuthology.task.install:The correct ceph version 18.2.4-843.g2c01458a is installed. 2024-09-13T14:13:05.550 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T14:13:05.550 INFO:teuthology.packaging:ref: None 2024-09-13T14:13:05.550 INFO:teuthology.packaging:tag: None 2024-09-13T14:13:05.551 INFO:teuthology.packaging:branch: reef 2024-09-13T14:13:05.551 INFO:teuthology.packaging:sha1: d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:13:05.551 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-13T14:13:05.824 DEBUG:teuthology.orchestra.run.smithi159:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-13T14:13:05.847 INFO:teuthology.orchestra.run.smithi159.stdout:18.2.4-843.g2c01458a.el9 2024-09-13T14:13:05.847 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-843.g2c01458a.el9 2024-09-13T14:13:05.848 INFO:teuthology.task.install:The correct ceph version 18.2.4-843.g2c01458a is installed. 2024-09-13T14:13:05.849 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T14:13:05.850 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:13:05.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:13:05.881 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:13:05.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T14:13:05.914 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T14:13:05.914 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:13:05.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:13:05.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:13:06.021 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:13:06.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T14:13:06.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T14:13:06.112 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T14:13:06.112 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:13:06.112 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:13:06.142 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:13:06.212 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:13:06.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T14:13:06.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T14:13:06.302 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T14:13:06.302 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:13:06.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:13:06.328 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:13:06.393 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:13:06.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T14:13:06.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T14:13:06.483 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T14:13:06.577 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'} 2024-09-13T14:13:06.577 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-09-13T14:13:06.578 INFO:tasks.cephadm:Cluster fsid is 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:06.578 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T14:13:06.578 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.42', 'mon.c': '[v2:172.21.15.42:3301,v1:172.21.15.42:6790]', 'mon.b': '172.21.15.159'} 2024-09-13T14:13:06.579 INFO:tasks.cephadm:First mon is mon.a on smithi042 2024-09-13T14:13:06.579 INFO:tasks.cephadm:First mgr is y 2024-09-13T14:13:06.579 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T14:13:06.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo hostname $(hostname -s) 2024-09-13T14:13:06.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo hostname $(hostname -s) 2024-09-13T14:13:06.632 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-09-13T14:13:06.632 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:13:06.776 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/83028/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-4697-gd58f7306', 'node_name': '172.21.4.75+confusa13', 'job_name': 'ceph-dev-new-build/ARCH=arm64,AVAILABLE_ARCH=arm64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-4697.gd58f7306'}, 'url': 'https://1.chacra.ceph.com/r/ceph/wip-aclamk-cbt-combined/d58f73069a8157a9ace12a0247ee3e8497707ea9/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-09-12 23:59:16.034150', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'wip-aclamk-cbt-combined', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/wip-aclamk-cbt-combined/d58f73069a8157a9ace12a0247ee3e8497707ea9/centos/9/flavors/default/', 'archs': ['source', 'x86_64', 'arm64'], 'distro': 'centos'}] 2024-09-13T14:13:07.014 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref reef, sha1 2c01458affee2b2435cd231b6e392435c1c21661 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-09-13T14:13:07.016 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/x86_64/flavors/default/cephadm 2024-09-13T14:13:07.016 INFO:tasks.cephadm:Downloading cephadm from url: https://1.chacra.ceph.com/binaries/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/x86_64/flavors/default/cephadm 2024-09-13T14:13:07.016 DEBUG:teuthology.orchestra.run.smithi042:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.369 INFO:teuthology.orchestra.run.smithi042.stdout:-rw-r--r--. 1 ubuntu ubuntu 217218 Sep 13 14:13 /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.369 DEBUG:teuthology.orchestra.run.smithi159:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/2c01458affee2b2435cd231b6e392435c1c21661/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.727 INFO:teuthology.orchestra.run.smithi159.stdout:-rw-r--r--. 1 ubuntu ubuntu 217218 Sep 13 14:13 /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.727 DEBUG:teuthology.orchestra.run.smithi042:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.745 DEBUG:teuthology.orchestra.run.smithi159:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T14:13:07.778 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-09-13T14:13:07.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-13T14:13:07.787 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-13T14:13:09.342 INFO:teuthology.orchestra.run.smithi159.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-13T14:13:09.426 INFO:teuthology.orchestra.run.smithi042.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph_version": "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)", 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout: "image_id": "c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5", 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout: "repo_digests": [ 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11" 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:13:39.349 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout: "ceph_version": "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)", 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout: "image_id": "c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5", 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout: "repo_digests": [ 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11" 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:13:45.672 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:13:45.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo mkdir -p /etc/ceph 2024-09-13T14:13:45.727 DEBUG:teuthology.orchestra.run.smithi159:> sudo mkdir -p /etc/ceph 2024-09-13T14:13:45.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 777 /etc/ceph 2024-09-13T14:13:45.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 777 /etc/ceph 2024-09-13T14:13:45.822 INFO:tasks.cephadm:Writing seed config... 2024-09-13T14:13:45.823 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T14:13:45.823 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T14:13:45.823 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T14:13:45.823 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T14:13:45.824 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T14:13:45.825 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:13:45.825 DEBUG:teuthology.orchestra.run.smithi042:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T14:13:45.854 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 4c2088c8-71da-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T14:13:45.855 DEBUG:teuthology.orchestra.run.smithi042:mon.a> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T14:13:45.897 DEBUG:teuthology.orchestra.run.smithi042:mgr.y> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T14:13:45.940 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T14:13:45.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.42 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:13:46.099 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T14:13:46.099 INFO:teuthology.orchestra.run.smithi042.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', '4c2088c8-71da-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.42', '--skip-admin-label'] 2024-09-13T14:13:46.123 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:13:46.123 INFO:teuthology.orchestra.run.smithi042.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T14:13:46.123 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying podman|docker is present... 2024-09-13T14:13:46.146 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:13:46.146 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying lvm2 is present... 2024-09-13T14:13:46.146 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying time synchronization is in place... 2024-09-13T14:13:46.154 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:13:46.154 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:13:46.160 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:13:46.160 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:13:46.167 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout enabled 2024-09-13T14:13:46.174 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout active 2024-09-13T14:13:46.174 INFO:teuthology.orchestra.run.smithi042.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:13:46.174 INFO:teuthology.orchestra.run.smithi042.stdout:Repeating the final host check... 2024-09-13T14:13:46.196 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stdout 5.2.2 2024-09-13T14:13:46.196 INFO:teuthology.orchestra.run.smithi042.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-13T14:13:46.196 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl is present 2024-09-13T14:13:46.197 INFO:teuthology.orchestra.run.smithi042.stdout:lvcreate is present 2024-09-13T14:13:46.203 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T14:13:46.203 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T14:13:46.209 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T14:13:46.209 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:13:46.216 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout enabled 2024-09-13T14:13:46.223 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout active 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Unit chronyd.service is enabled and running 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Host looks OK 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Cluster fsid: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Acquiring lock 140708769480272 on /run/cephadm/4c2088c8-71da-11ef-bceb-c7b262605968.lock 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Lock 140708769480272 acquired on /run/cephadm/4c2088c8-71da-11ef-bceb-c7b262605968.lock 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying IP 172.21.15.42 port 3300 ... 2024-09-13T14:13:46.224 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying IP 172.21.15.42 port 6789 ... 2024-09-13T14:13:46.225 INFO:teuthology.orchestra.run.smithi042.stdout:Base mon IP(s) is [172.21.15.42:3300, 172.21.15.42:6789], mon addrv is [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-09-13T14:13:46.227 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.42 metric 100 2024-09-13T14:13:46.227 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.42 metric 100 2024-09-13T14:13:46.230 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T14:13:46.230 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T14:13:46.230 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1786sec hoplimit 64 pref medium 2024-09-13T14:13:46.233 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T14:13:46.233 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:1440/64 scope link 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:Mon IP `172.21.15.42` is in CIDR network `172.21.0.0/20` 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:Mon IP `172.21.15.42` is in CIDR network `172.21.0.0/20` 2024-09-13T14:13:46.234 INFO:teuthology.orchestra.run.smithi042.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-13T14:13:46.235 INFO:teuthology.orchestra.run.smithi042.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T14:13:46.235 INFO:teuthology.orchestra.run.smithi042.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stdout c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Getting image source signatures 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Copying blob sha256:f9a1328d66f49e17fe78ed666f97dd3c34cc07be60bd7ae6ba8f2402a2a33f7f 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Copying config sha256:c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 2024-09-13T14:13:46.717 INFO:teuthology.orchestra.run.smithi042.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-13T14:13:47.571 INFO:teuthology.orchestra.run.smithi042.stdout:ceph: stdout ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable) 2024-09-13T14:13:47.572 INFO:teuthology.orchestra.run.smithi042.stdout:Ceph version: ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable) 2024-09-13T14:13:47.572 INFO:teuthology.orchestra.run.smithi042.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T14:13:48.510 INFO:teuthology.orchestra.run.smithi042.stdout:stat: stdout 167 167 2024-09-13T14:13:48.510 INFO:teuthology.orchestra.run.smithi042.stdout:Creating initial keys... 2024-09-13T14:13:49.599 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph-authtool: stdout AQCcSORmRyFfORAAB8T0rZ7RF1TIkqd3OiQCWg== 2024-09-13T14:13:50.639 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph-authtool: stdout AQCdSORmBwm0ORAAXha9KahuPcvIiRMkbHw3Tw== 2024-09-13T14:13:51.612 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph-authtool: stdout AQCfSORmo+d2ARAAq/8Sc9+qta0leR2+ygQ6vQ== 2024-09-13T14:13:51.613 INFO:teuthology.orchestra.run.smithi042.stdout:Creating initial monmap... 2024-09-13T14:13:52.575 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:13:52.575 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-09-13T14:13:52.575 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:52.575 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout:monmaptool for a [v2:172.21.15.42:3300,v1:172.21.15.42:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout:setting min_mon_release = pacific 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: set fsid to 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:13:52.576 INFO:teuthology.orchestra.run.smithi042.stdout:Creating mon... 2024-09-13T14:13:53.540 INFO:teuthology.orchestra.run.smithi042.stdout:create mon.a on 2024-09-13T14:13:53.783 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-13T14:13:53.936 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T14:13:54.137 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target. 2024-09-13T14:13:54.137 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target. 2024-09-13T14:13:54.366 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a 2024-09-13T14:13:54.366 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Failed to reset failed state of unit ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T14:13:54.533 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target.wants/ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968@.service. 2024-09-13T14:13:54.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:54 smithi042 podman[28844]: 2024-09-13 14:13:54.678373703 +0000 UTC m=+0.016251491 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph:reef 2024-09-13T14:13:55.029 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:13:55.029 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout disabled 2024-09-13T14:13:55.036 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:13:55.036 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:13:55.037 INFO:teuthology.orchestra.run.smithi042.stdout:firewalld.service is not enabled 2024-09-13T14:13:55.037 INFO:teuthology.orchestra.run.smithi042.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:13:55.037 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mon to start... 2024-09-13T14:13:55.037 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mon... 2024-09-13T14:13:55.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:54 smithi042 podman[28844]: 2024-09-13 14:13:54.932322199 +0000 UTC m=+0.270199990 container init 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T14:13:55.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:54 smithi042 podman[28844]: 2024-09-13 14:13:54.936863953 +0000 UTC m=+0.274741732 container start 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-2c01458) 2024-09-13T14:13:55.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:54 smithi042 ceph-mon[28881]: mkfs 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:55.252 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:54 smithi042 ceph-mon[28881]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:13:55.252 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 bash[28844]: 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 2024-09-13T14:13:55.252 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 systemd[1]: Started Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:13:56.082 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:13:56.082 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: monmap e1: 1 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:13:56.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: fsmap 2024-09-13T14:13:56.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:13:56.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: mgrmap e1: no daemons active 2024-09-13T14:13:56.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:55 smithi042 ceph-mon[28881]: from='client.? 172.21.15.42:0/2307060768' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T14:13:56.084 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout id: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout services: 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.554994s) 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout data: 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T14:13:56.085 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T14:13:56.086 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T14:13:56.086 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:56.086 INFO:teuthology.orchestra.run.smithi042.stdout:mon is available 2024-09-13T14:13:56.086 INFO:teuthology.orchestra.run.smithi042.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout fsid = 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:13:57.065 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:13:57.066 INFO:teuthology.orchestra.run.smithi042.stdout:Generating new minimal ceph.conf... 2024-09-13T14:13:57.097 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:56 smithi042 ceph-mon[28881]: from='client.? 172.21.15.42:0/3652449843' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:13:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:56 smithi042 ceph-mon[28881]: from='client.? 172.21.15.42:0/3652449843' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T14:13:58.095 INFO:teuthology.orchestra.run.smithi042.stdout:Restarting the monitor... 2024-09-13T14:13:58.110 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:57 smithi042 ceph-mon[28881]: from='client.? 172.21.15.42:0/3525299991' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:13:58.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 systemd[1]: Stopping Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:13:58.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[28858]: 2024-09-13T14:13:58.355+0000 7f88cb61e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:13:58.500 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[28858]: 2024-09-13T14:13:58.355+0000 7f88cb61e640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T14:13:58.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 podman[29199]: 2024-09-13 14:13:58.49916654 +0000 UTC m=+0.242545602 container died 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:13:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 podman[29199]: 2024-09-13 14:13:58.861374178 +0000 UTC m=+0.604753279 container cleanup 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T14:13:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:58 smithi042 bash[29199]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:13:59.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:59 smithi042 podman[29216]: 2024-09-13 14:13:59.103660047 +0000 UTC m=+0.602668098 container remove 80104123345a51e644402f4e8de0cf891782da3d881fe888a3f2158c5128e410 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=reef-2c01458, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T14:13:59.447 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:59 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:13:59.447 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:59 smithi042 systemd[1]: Stopped Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:13:59.447 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:59 smithi042 systemd[1]: Starting Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:13:59.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:13:59 smithi042 podman[29312]: 2024-09-13 14:13:59.348933921 +0000 UTC m=+0.016386777 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph:reef 2024-09-13T14:14:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:00 smithi042 podman[29312]: 2024-09-13 14:14:00.673148216 +0000 UTC m=+1.340601072 container create 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458, GIT_BRANCH=HEAD) 2024-09-13T14:14:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 podman[29312]: 2024-09-13 14:14:02.557229254 +0000 UTC m=+3.224682109 container init 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-13T14:14:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 podman[29312]: 2024-09-13 14:14:02.563817159 +0000 UTC m=+3.231270018 container start 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.build-date=20240731) 2024-09-13T14:14:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:14:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable), process ceph-mon, pid 7 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: pidfile_write: ignore empty --pid-file 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: load: jerasure load: lrc 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Git sha 0 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Compile date 2024-09-12 19:46:18 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: DB SUMMARY 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: DB Session ID: 910ZFNNTM44VESDCH2B3 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:14:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84546 ; 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.env: 0x55f2a8facbc0 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.info_log: 0x55f2a9f3cbe0 2024-09-13T14:14:02.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.statistics: (nil) 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.use_fsync: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:14:02.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.db_log_dir: 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.wal_dir: 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:14:02.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.write_buffer_manager: 0x55f2a9f4a320 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.unordered_write: 0 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:14:02.854 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.row_cache: None 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.wal_filter: None 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.wal_compression: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:14:02.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:14:02.856 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_open_files: -1 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:14:02.857 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Compression algorithms supported: 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kZSTD supported: 0 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:14:02.858 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.merge_operator: 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_filter: None 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f2a9f3c7e0) 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:14:02.859 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_type: 0 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_index_type: 0 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_shortening: 1 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: checksum: 4 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: no_block_cache: 0 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache: 0x55f2a9f31350 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_options: 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: capacity : 536870912 2024-09-13T14:14:02.860 INFO:journalctl@ceph.mon.a.smithi042.stdout: num_shard_bits : 4 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: persistent_cache: (nil) 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size: 4096 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size_deviation: 10 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_restart_interval: 16 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: metadata_block_size: 4096 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: partition_filters: 0 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: use_delta_encoding: 1 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: filter_policy: bloomfilter 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: whole_key_filtering: 1 2024-09-13T14:14:02.861 INFO:journalctl@ceph.mon.a.smithi042.stdout: verify_compression: 0 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: format_version: 5 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_index_compression: 1 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_align: 0 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: max_auto_readahead_size: 262144 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: prepopulate_block_cache: 0 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:14:02.862 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression: NoCompression 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.num_levels: 7 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:14:02.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:14:02.864 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:14:02.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:14:02.866 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:14:02.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:14:02.868 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.ttl: 2592000 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:14:02.869 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e71d50f8-8d31-418b-b47a-35930c58253a 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236842593523, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236842594508, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81096, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79246, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73447, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726236842, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e71d50f8-8d31-418b-b47a-35930c58253a", "db_session_id": "910ZFNNTM44VESDCH2B3", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236842594623, "job": 1, "event": "recovery_finished"} 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f2a9f5ce00 2024-09-13T14:14:02.870 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: DB pointer 0x55f2aa050000 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** DB Stats ** 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:14:02.871 INFO:journalctl@ceph.mon.a.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: L0 2/0 81.06 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: Sum 2/0 81.06 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 123.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 123.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.872 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative compaction: 0.00 GB write, 22.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval compaction: 0.00 GB write, 22.01 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Block cache BinnedLRUCache@0x55f2a9f31350#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: starting mon.a rank 0 at public addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] at bind addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:02.873 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???) e1 preinit fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).mds e1 new map 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).mds e1 print_map 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: e1 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: legacy client fscid: -1 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout: No filesystems configured 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T14:14:02.874 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T14:14:02.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: monmap e1: 1 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:14:02.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: fsmap 2024-09-13T14:14:02.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T14:14:02.875 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:02 smithi042 ceph-mon[29349]: mgrmap e1: no daemons active 2024-09-13T14:14:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-13T14:14:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:03 smithi042 bash[29312]: 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 2024-09-13T14:14:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:03 smithi042 systemd[1]: Started Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:14:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:07 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3608850432' entity='client.admin' 2024-09-13T14:14:10.939 INFO:teuthology.orchestra.run.smithi042.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T14:14:10.960 INFO:teuthology.orchestra.run.smithi042.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:14:10.960 INFO:teuthology.orchestra.run.smithi042.stdout:Creating mgr... 2024-09-13T14:14:10.960 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T14:14:10.960 INFO:teuthology.orchestra.run.smithi042.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T14:14:11.221 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y 2024-09-13T14:14:11.221 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Failed to reset failed state of unit ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T14:14:11.457 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968.target.wants/ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-4c2088c8-71da-11ef-bceb-c7b262605968@.service. 2024-09-13T14:14:11.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 systemd[1]: Starting Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:14:11.989 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 podman[29595]: 2024-09-13 14:14:11.686934181 +0000 UTC m=+0.104023915 container create f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-13T14:14:11.989 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 podman[29595]: 2024-09-13 14:14:11.599119124 +0000 UTC m=+0.016208826 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph:reef 2024-09-13T14:14:11.990 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 podman[29595]: 2024-09-13 14:14:11.914322969 +0000 UTC m=+0.331412670 container init f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T14:14:11.990 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 podman[29595]: 2024-09-13 14:14:11.919025141 +0000 UTC m=+0.336114833 container start f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:14:11.990 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 bash[29595]: f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 2024-09-13T14:14:12.006 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:14:12.006 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout disabled 2024-09-13T14:14:12.015 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:14:12.015 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:14:12.015 INFO:teuthology.orchestra.run.smithi042.stdout:firewalld.service is not enabled 2024-09-13T14:14:12.015 INFO:teuthology.orchestra.run.smithi042.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T14:14:12.026 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:14:12.026 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout disabled 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:firewalld.service is not enabled 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mgr to start... 2024-09-13T14:14:12.035 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mgr... 2024-09-13T14:14:12.277 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:11 smithi042 systemd[1]: Started Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:14:12.277 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.054+0000 7f976895b040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:14:12.277 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.114+0000 7f976895b040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:12.277 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.275+0000 7f976895b040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:14:12.551 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.350+0000 7f976895b040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:14:12.551 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.504+0000 7f976895b040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:12.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1276456516' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:14:12.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.656+0000 7f976895b040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:14:13.239 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:12.982+0000 7f976895b040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:14:13.239 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:13.046+0000 7f976895b040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsid": "4c2088c8-71da-11ef-bceb-c7b262605968", 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:14:13.359 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-13T14:14:13.360 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:14:13.361 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:14:13.364 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:14:13.364 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:14:13.364 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.364 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:14:13.364 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.365 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:13:54.974056+0000", 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:13.366 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:13.367 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:14:13.367 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:13.367 INFO:teuthology.orchestra.run.smithi042.stdout:mgr not available, waiting (1/15)... 2024-09-13T14:14:13.597 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:13.238+0000 7f976895b040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:14:14.597 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:14 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:14.154+0000 7f976895b040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:14:14.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:14 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:14.223+0000 7f976895b040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:14:15.258 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:14 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:14.988+0000 7f976895b040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:14:15.258 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.109+0000 7f976895b040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:14:15.258 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.257+0000 7f976895b040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:15.512 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.321+0000 7f976895b040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:14:15.513 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.479+0000 7f976895b040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:14:15.780 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.551+0000 7f976895b040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:14:15.780 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.614+0000 7f976895b040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:14:15.780 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.778+0000 7f976895b040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:14:16.061 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.847+0000 7f976895b040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:14:16.061 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:15.919+0000 7f976895b040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:14:16.062 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:16 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:16.008+0000 7f976895b040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:16 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2196212413' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:14:16.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:16 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:16.071+0000 7f976895b040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:14:16.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:16 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:16.134+0000 7f976895b040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:14:16.836 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:16.836 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:16.836 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsid": "4c2088c8-71da-11ef-bceb-c7b262605968", 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2024-09-13T14:14:16.837 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:14:16.838 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:14:16.841 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:14:16.841 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:14:16.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:16.843 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:13:54.974056+0000", 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:16.844 INFO:teuthology.orchestra.run.smithi042.stdout:mgr not available, waiting (2/15)... 2024-09-13T14:14:17.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:16 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:16.963+0000 7f976895b040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: Activating manager daemon y 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: mgrmap e2: y(active, starting, since 0.00292343s) 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:14:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: Manager daemon y is now available 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' 2024-09-13T14:14:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:17 smithi042 ceph-mon[29349]: from='mgr.14100 172.21.15.42:0/455818115' entity='mgr.y' 2024-09-13T14:14:19.218 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:18 smithi042 ceph-mon[29349]: mgrmap e3: y(active, since 1.00709s) 2024-09-13T14:14:20.155 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:19 smithi042 ceph-mon[29349]: mgrmap e4: y(active, since 2s) 2024-09-13T14:14:20.155 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:19 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1810512175' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T14:14:20.162 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:20.162 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsid": "4c2088c8-71da-11ef-bceb-c7b262605968", 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 0 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T14:14:20.163 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "quorum_age": 16, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T14:14:20.164 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T14:14:20.165 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ], 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T14:14:20.166 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T14:13:54.974056+0000", 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout }, 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:20.167 INFO:teuthology.orchestra.run.smithi042.stdout:mgr is available 2024-09-13T14:14:21.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:20 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/456508832' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout fsid = 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T14:14:21.384 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T14:14:21.385 INFO:teuthology.orchestra.run.smithi042.stdout:Enabling cephadm module... 2024-09-13T14:14:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3924877675' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T14:14:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setuser ceph since I am not root 2024-09-13T14:14:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setgroup ceph since I am not root 2024-09-13T14:14:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.202+0000 7f93432dc040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:14:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.261+0000 7f93432dc040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:22.640 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.416+0000 7f93432dc040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:14:22.640 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.484+0000 7f93432dc040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:14:22.641 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.639+0000 7f93432dc040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:22.931 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:22.785+0000 7f93432dc040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:14:23.244 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:23 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3924877675' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T14:14:23.244 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:23 smithi042 ceph-mon[29349]: mgrmap e5: y(active, since 5s) 2024-09-13T14:14:23.244 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:23.113+0000 7f93432dc040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:14:23.244 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:23.176+0000 7f93432dc040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:14:23.568 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:23.368+0000 7f93432dc040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:14:23.846 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for the mgr to restart... 2024-09-13T14:14:23.847 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mgr epoch 5... 2024-09-13T14:14:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:24 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2040861292' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:14:24.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:24.288+0000 7f93432dc040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:14:24.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:24.356+0000 7f93432dc040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:14:25.365 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.098+0000 7f93432dc040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:14:25.365 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.217+0000 7f93432dc040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:14:25.650 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.363+0000 7f93432dc040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:25.650 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.426+0000 7f93432dc040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:14:25.650 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.578+0000 7f93432dc040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:14:25.650 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.648+0000 7f93432dc040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:14:25.940 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.710+0000 7f93432dc040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:14:25.940 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.872+0000 7f93432dc040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:14:25.940 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:25.939+0000 7f93432dc040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:14:26.216 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:26 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:26.007+0000 7f93432dc040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:14:26.216 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:26 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:26.093+0000 7f93432dc040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:26.217 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:26 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:26.154+0000 7f93432dc040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:14:26.217 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:26 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:26.215+0000 7f93432dc040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:14:27.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:27.024+0000 7f93432dc040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:14:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: Active manager daemon y restarted 2024-09-13T14:14:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: Activating manager daemon y 2024-09-13T14:14:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T14:14:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: mgrmap e6: y(active, starting, since 0.00322859s) 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: Manager daemon y is now available 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:14:27.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:27 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:14:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:28 smithi042 ceph-mon[29349]: Found migration_current of "None". Setting to last migration. 2024-09-13T14:14:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:28 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:28 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:28 smithi042 ceph-mon[29349]: mgrmap e7: y(active, since 1.00549s) 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:mgr epoch 5 is available 2024-09-13T14:14:28.619 INFO:teuthology.orchestra.run.smithi042.stdout:Setting orchestrator backend to cephadm... 2024-09-13T14:14:29.285 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:29 smithi042 ceph-mon[29349]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:14:29.285 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:29 smithi042 ceph-mon[29349]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:14:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-mon[29349]: mgrmap e8: y(active, since 2s) 2024-09-13T14:14:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:30.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:30.293+0000 7f92b07c7640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:14:30] ENGINE Error in HTTPServer.serve 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Traceback (most recent call last): 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._connections.run(self.expiration_interval) 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._run(expiration_interval) 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s = self.context.wrap_socket( 2024-09-13T14:14:30.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: return self.sslsocket_class._create( 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self.do_handshake() 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._sslobj.do_handshake() 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:14:30.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:30.957 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T14:14:30.957 INFO:teuthology.orchestra.run.smithi042.stdout:Generating ssh key... 2024-09-13T14:14:31.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:31 smithi042 ceph-mon[29349]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:31.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:31 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:30] ENGINE Bus STARTING 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:30] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:30] ENGINE Error in HTTPServer.serve 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:14:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:14:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:30] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:30] ENGINE Bus STARTED 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-mon[29349]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Generating public/private rsa key pair. 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Your identification has been saved in /tmp/tmph2vrgcf6/key 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Your public key has been saved in /tmp/tmph2vrgcf6/key.pub 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: The key fingerprint is: 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: SHA256:F/RBcQmJ8snCEzYIbpMILLagW4Z9OqMo3p7pysImECU ceph-4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: The key's randomart image is: 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: +---[RSA 3072]----+ 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |o .. . .o=+.. | 2024-09-13T14:14:32.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |E+.o .. =....o. | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |==o = o *... | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |+.+... + +. | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: | = o So. | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |o + . | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |+. o | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |Bo. o | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: |*++* | 2024-09-13T14:14:32.351 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: +----[SHA256]-----+ 2024-09-13T14:14:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:33 smithi042 ceph-mon[29349]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:33 smithi042 ceph-mon[29349]: Generating ssh key... 2024-09-13T14:14:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:33 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:33 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:33.928 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk20ePAIl7ywe/ZfE/SzMJUAxYFMswKkVqiMDmFobVooTQQjtMcO1l1PllO+6UYagcCBM2kdGG0iKPIKGTW6qqRhiMFq+gZqEmY0CJr4iXe1q6UUULFWRTaWBSqhb9XdC+Q5f5ip8wJ1tzymc3xRhPYJU47ZQ+qBMwNKxgw/OooJG703BA8YuPKQuofxQ9kcAQ6K/YwmfxGoov20dj58atZ2f//N4Kh0aC7YxdT9ioxvHNmBfiUhzX/if1sHttHRJEYniGAT6rQSn1mKuQK2Q0EzKxW0biovZEKTd5WszMgpKx/4mHGeYNBvBKEK7zRYi2kQNqv3JCgQ00DRpe6ZqKYmJlulqf7F5JiAtkbbRlxH061meQdnKjF5cRZEeyIihgF2u49lR5swLvKCNEJvcRNGXRDvQzd1tPHcH8PRhYoIFiVWYdaJ/96H/n4PGZbFVtVMT50A3vuXg/1Gqg5EMyIvQl+rE+9GaCZjCrJ3HZXoGn+glT05pSgrBqwNIWKy8= ceph-4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:33.929 INFO:teuthology.orchestra.run.smithi042.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T14:14:33.929 INFO:teuthology.orchestra.run.smithi042.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T14:14:33.940 INFO:teuthology.orchestra.run.smithi042.stdout:Adding host smithi042... 2024-09-13T14:14:34.198 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:34 smithi042 ceph-mon[29349]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:35.311 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:35 smithi042 ceph-mon[29349]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi042", "addr": "172.21.15.42", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:36.240 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:36 smithi042 ceph-mon[29349]: Deploying cephadm binary to smithi042 2024-09-13T14:14:38.520 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout Added host 'smithi042' with addr '172.21.15.42' 2024-09-13T14:14:38.520 INFO:teuthology.orchestra.run.smithi042.stdout:Deploying unmanaged mon service... 2024-09-13T14:14:39.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:38 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:39.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:38 smithi042 ceph-mon[29349]: Added host smithi042 2024-09-13T14:14:39.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:38 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:40 smithi042 ceph-mon[29349]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:40 smithi042 ceph-mon[29349]: Saving service mon spec with placement count:5 2024-09-13T14:14:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:40 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:40 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:40.499 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T14:14:40.499 INFO:teuthology.orchestra.run.smithi042.stdout:Deploying unmanaged mgr service... 2024-09-13T14:14:41.813 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:41 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:41.813 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:41 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:41.815 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T14:14:42.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:42 smithi042 ceph-mon[29349]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:42.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:42 smithi042 ceph-mon[29349]: Saving service mgr spec with placement count:2 2024-09-13T14:14:42.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:42 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3719642744' entity='client.admin' 2024-09-13T14:14:44.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:43 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:44.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:43 smithi042 ceph-mon[29349]: from='mgr.14118 172.21.15.42:0/1663502281' entity='mgr.y' 2024-09-13T14:14:44.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:43 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2755104254' entity='client.admin' 2024-09-13T14:14:44.448 INFO:teuthology.orchestra.run.smithi042.stdout:Enabling the dashboard module... 2024-09-13T14:14:45.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2646797685' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T14:14:45.361 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setuser ceph since I am not root 2024-09-13T14:14:45.361 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setgroup ceph since I am not root 2024-09-13T14:14:45.361 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.300+0000 7f4c0d144040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:14:45.361 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.359+0000 7f4c0d144040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:45.734 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.513+0000 7f4c0d144040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:14:45.734 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.581+0000 7f4c0d144040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:14:45.734 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.732+0000 7f4c0d144040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:46.089 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:45.879+0000 7f4c0d144040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:14:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:46 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2646797685' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T14:14:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:46 smithi042 ceph-mon[29349]: mgrmap e9: y(active, since 18s) 2024-09-13T14:14:46.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:46 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:46.205+0000 7f4c0d144040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:14:46.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:46 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:46.269+0000 7f4c0d144040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:14:46.724 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:46 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:46.460+0000 7f4c0d144040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:14:47.462 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:47 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3780843769' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T14:14:47.462 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:47.391+0000 7f4c0d144040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:14:47.773 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:47.461+0000 7f4c0d144040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:14:47.775 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:47.775 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for the mgr to restart... 2024-09-13T14:14:47.776 INFO:teuthology.orchestra.run.smithi042.stdout:Waiting for mgr epoch 9... 2024-09-13T14:14:48.486 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.218+0000 7f4c0d144040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:14:48.486 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.339+0000 7f4c0d144040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:14:48.771 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.485+0000 7f4c0d144040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:14:48.771 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.548+0000 7f4c0d144040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:14:48.772 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.700+0000 7f4c0d144040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:14:48.772 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.770+0000 7f4c0d144040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:14:49.062 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.832+0000 7f4c0d144040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:14:49.063 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:48.994+0000 7f4c0d144040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:14:49.063 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:49.061+0000 7f4c0d144040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:14:49.339 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:49.130+0000 7f4c0d144040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:14:49.339 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:49.215+0000 7f4c0d144040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:14:49.339 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:49.277+0000 7f4c0d144040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:14:49.339 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:49.338+0000 7f4c0d144040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:14:50.416 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:50.145+0000 7f4c0d144040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:14:50.416 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: Active manager daemon y restarted 2024-09-13T14:14:50.416 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: Activating manager daemon y 2024-09-13T14:14:50.416 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T14:14:50.416 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: mgrmap e10: y(active, starting, since 0.00290063s) 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: Manager daemon y is now available 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:14:50.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:14:51.950 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:51.768+0000 7f4b735ce640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:14:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:14:51.950 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Traceback (most recent call last): 2024-09-13T14:14:51.950 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:14:51.950 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._connections.run(self.expiration_interval) 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._run(expiration_interval) 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s = self.context.wrap_socket( 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: return self.sslsocket_class._create( 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self.do_handshake() 2024-09-13T14:14:51.951 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:14:51.952 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._sslobj.do_handshake() 2024-09-13T14:14:51.952 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:14:51.952 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:14:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout { 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout } 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:mgr epoch 9 is available 2024-09-13T14:14:51.953 INFO:teuthology.orchestra.run.smithi042.stdout:Generating a dashboard self-signed certificate... 2024-09-13T14:14:52.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:52 smithi042 ceph-mon[29349]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T14:14:52.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:52 smithi042 ceph-mon[29349]: mgrmap e11: y(active, since 1.00534s) 2024-09-13T14:14:52.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:52 smithi042 ceph-mon[29349]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T14:14:53.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:51] ENGINE Bus STARTING 2024-09-13T14:14:53.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:51] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:14:53.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:51] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:14:53.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:51] ENGINE Bus STARTED 2024-09-13T14:14:53.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: [13/Sep/2024:14:14:51] ENGINE Error in HTTPServer.serve 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:14:53.291 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: mgrmap e12: y(active, since 2s) 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:14:53.292 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:14:53.293 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T14:14:53.293 INFO:teuthology.orchestra.run.smithi042.stdout:Creating initial admin user... 2024-09-13T14:14:54.415 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:54 smithi042 ceph-mon[29349]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:55 smithi042 ceph-mon[29349]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:14:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:14:55.355 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$V.ltl1kDuL/2jZ.ILF/Pb.zQmAZtUXGgLazJyGp45xhVroX2WYgde", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726236894, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T14:14:55.355 INFO:teuthology.orchestra.run.smithi042.stdout:Fetching dashboard port number... 2024-09-13T14:14:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:56 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3430690176' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T14:14:56.454 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T14:14:56.464 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-13T14:14:56.464 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout disabled 2024-09-13T14:14:56.472 INFO:teuthology.orchestra.run.smithi042.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-13T14:14:56.472 INFO:teuthology.orchestra.run.smithi042.stdout:systemctl: stdout inactive 2024-09-13T14:14:56.472 INFO:teuthology.orchestra.run.smithi042.stdout:firewalld.service is not enabled 2024-09-13T14:14:56.472 INFO:teuthology.orchestra.run.smithi042.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout:Ceph Dashboard is now available at: 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout: URL: https://smithi042.front.sepia.ceph.com:8443/ 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout: User: admin 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout: Password: mhtnzwxteb 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:56.474 INFO:teuthology.orchestra.run.smithi042.stdout:Saving cluster configuration to /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config directory 2024-09-13T14:14:56.475 INFO:teuthology.orchestra.run.smithi042.stdout:Enabling autotune for osd_memory_target 2024-09-13T14:14:57.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:57 smithi042 ceph-mon[29349]: mgrmap e13: y(active, since 6s) 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.842 INFO:teuthology.orchestra.run.smithi042.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: ceph telemetry on 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout:For more information see: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:14:58.843 INFO:teuthology.orchestra.run.smithi042.stdout:Bootstrap complete. 2024-09-13T14:14:58.880 INFO:tasks.cephadm:Fetching config... 2024-09-13T14:14:58.880 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:14:58.880 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T14:14:58.898 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T14:14:58.898 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:14:58.898 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T14:14:58.954 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T14:14:58.955 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:14:58.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T14:14:59.021 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T14:14:59.022 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:14:59.022 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T14:14:59.078 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T14:14:59.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk20ePAIl7ywe/ZfE/SzMJUAxYFMswKkVqiMDmFobVooTQQjtMcO1l1PllO+6UYagcCBM2kdGG0iKPIKGTW6qqRhiMFq+gZqEmY0CJr4iXe1q6UUULFWRTaWBSqhb9XdC+Q5f5ip8wJ1tzymc3xRhPYJU47ZQ+qBMwNKxgw/OooJG703BA8YuPKQuofxQ9kcAQ6K/YwmfxGoov20dj58atZ2f//N4Kh0aC7YxdT9ioxvHNmBfiUhzX/if1sHttHRJEYniGAT6rQSn1mKuQK2Q0EzKxW0biovZEKTd5WszMgpKx/4mHGeYNBvBKEK7zRYi2kQNqv3JCgQ00DRpe6ZqKYmJlulqf7F5JiAtkbbRlxH061meQdnKjF5cRZEeyIihgF2u49lR5swLvKCNEJvcRNGXRDvQzd1tPHcH8PRhYoIFiVWYdaJ/96H/n4PGZbFVtVMT50A3vuXg/1Gqg5EMyIvQl+rE+9GaCZjCrJ3HZXoGn+glT05pSgrBqwNIWKy8= ceph-4c2088c8-71da-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:14:59.164 INFO:teuthology.orchestra.run.smithi042.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk20ePAIl7ywe/ZfE/SzMJUAxYFMswKkVqiMDmFobVooTQQjtMcO1l1PllO+6UYagcCBM2kdGG0iKPIKGTW6qqRhiMFq+gZqEmY0CJr4iXe1q6UUULFWRTaWBSqhb9XdC+Q5f5ip8wJ1tzymc3xRhPYJU47ZQ+qBMwNKxgw/OooJG703BA8YuPKQuofxQ9kcAQ6K/YwmfxGoov20dj58atZ2f//N4Kh0aC7YxdT9ioxvHNmBfiUhzX/if1sHttHRJEYniGAT6rQSn1mKuQK2Q0EzKxW0biovZEKTd5WszMgpKx/4mHGeYNBvBKEK7zRYi2kQNqv3JCgQ00DRpe6ZqKYmJlulqf7F5JiAtkbbRlxH061meQdnKjF5cRZEeyIihgF2u49lR5swLvKCNEJvcRNGXRDvQzd1tPHcH8PRhYoIFiVWYdaJ/96H/n4PGZbFVtVMT50A3vuXg/1Gqg5EMyIvQl+rE+9GaCZjCrJ3HZXoGn+glT05pSgrBqwNIWKy8= ceph-4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:59.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk20ePAIl7ywe/ZfE/SzMJUAxYFMswKkVqiMDmFobVooTQQjtMcO1l1PllO+6UYagcCBM2kdGG0iKPIKGTW6qqRhiMFq+gZqEmY0CJr4iXe1q6UUULFWRTaWBSqhb9XdC+Q5f5ip8wJ1tzymc3xRhPYJU47ZQ+qBMwNKxgw/OooJG703BA8YuPKQuofxQ9kcAQ6K/YwmfxGoov20dj58atZ2f//N4Kh0aC7YxdT9ioxvHNmBfiUhzX/if1sHttHRJEYniGAT6rQSn1mKuQK2Q0EzKxW0biovZEKTd5WszMgpKx/4mHGeYNBvBKEK7zRYi2kQNqv3JCgQ00DRpe6ZqKYmJlulqf7F5JiAtkbbRlxH061meQdnKjF5cRZEeyIihgF2u49lR5swLvKCNEJvcRNGXRDvQzd1tPHcH8PRhYoIFiVWYdaJ/96H/n4PGZbFVtVMT50A3vuXg/1Gqg5EMyIvQl+rE+9GaCZjCrJ3HZXoGn+glT05pSgrBqwNIWKy8= ceph-4c2088c8-71da-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T14:14:59.224 INFO:teuthology.orchestra.run.smithi159.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCk20ePAIl7ywe/ZfE/SzMJUAxYFMswKkVqiMDmFobVooTQQjtMcO1l1PllO+6UYagcCBM2kdGG0iKPIKGTW6qqRhiMFq+gZqEmY0CJr4iXe1q6UUULFWRTaWBSqhb9XdC+Q5f5ip8wJ1tzymc3xRhPYJU47ZQ+qBMwNKxgw/OooJG703BA8YuPKQuofxQ9kcAQ6K/YwmfxGoov20dj58atZ2f//N4Kh0aC7YxdT9ioxvHNmBfiUhzX/if1sHttHRJEYniGAT6rQSn1mKuQK2Q0EzKxW0biovZEKTd5WszMgpKx/4mHGeYNBvBKEK7zRYi2kQNqv3JCgQ00DRpe6ZqKYmJlulqf7F5JiAtkbbRlxH061meQdnKjF5cRZEeyIihgF2u49lR5swLvKCNEJvcRNGXRDvQzd1tPHcH8PRhYoIFiVWYdaJ/96H/n4PGZbFVtVMT50A3vuXg/1Gqg5EMyIvQl+rE+9GaCZjCrJ3HZXoGn+glT05pSgrBqwNIWKy8= ceph-4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:14:59.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T14:14:59.435 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:14:59.557 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:14:59 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3609534865' entity='client.admin' 2024-09-13T14:15:02.568 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T14:15:02.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T14:15:03.133 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:02 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/640980310' entity='client.admin' 2024-09-13T14:15:03.252 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi042", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:04 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:05.764 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi159 2024-09-13T14:15:05.764 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:15:05.764 DEBUG:teuthology.orchestra.run.smithi159:> dd of=/etc/ceph/ceph.conf 2024-09-13T14:15:05.784 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:15:05.784 DEBUG:teuthology.orchestra.run.smithi159:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:05.842 INFO:tasks.cephadm:Adding host smithi159 to orchestrator... 2024-09-13T14:15:05.843 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch host add smithi159 2024-09-13T14:15:06.043 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:06.273 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:06.273 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:06.273 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:06.273 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:06.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:06.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:15:06.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:06.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:06.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:07 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:07 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:07 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:15:08.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:08 smithi042 ceph-mon[29349]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi159", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:09.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:09 smithi042 ceph-mon[29349]: Deploying cephadm binary to smithi159 2024-09-13T14:15:11.089 INFO:teuthology.orchestra.run.smithi042.stdout:Added host 'smithi159' with addr '172.21.15.159' 2024-09-13T14:15:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:11 smithi042 ceph-mon[29349]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:11 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:11 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:11.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T14:15:11.899 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:12 smithi042 ceph-mon[29349]: Added host smithi159 2024-09-13T14:15:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:13.271 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:13 smithi042 ceph-mon[29349]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:13.271 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:13 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:13.669 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:15:13.669 INFO:teuthology.orchestra.run.smithi042.stdout:[{"addr": "172.21.15.42", "hostname": "smithi042", "labels": [], "status": ""}, {"addr": "172.21.15.159", "hostname": "smithi159", "labels": [], "status": ""}] 2024-09-13T14:15:14.223 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T14:15:14.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T14:15:14.418 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:15.515 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:15.516 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:16 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:15:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:16 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:16 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:16 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:17.161 INFO:teuthology.orchestra.run.smithi042.stderr:adjusted tunables profile to default 2024-09-13T14:15:17.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:17 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:17.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:17 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:17.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:17 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:15:17.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:17 smithi042 ceph-mon[29349]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:17.473 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:17 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/640632234' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:15:17.809 INFO:tasks.cephadm:Adding mon.a on smithi042 2024-09-13T14:15:17.809 INFO:tasks.cephadm:Adding mon.c on smithi042 2024-09-13T14:15:17.809 INFO:tasks.cephadm:Adding mon.b on smithi159 2024-09-13T14:15:17.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi159:172.21.15.159=b' 2024-09-13T14:15:18.009 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:18 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/640632234' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:15:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:18 smithi042 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:19.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:19 smithi042 ceph-mon[29349]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:19.723 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled mon update... 2024-09-13T14:15:20.326 DEBUG:teuthology.orchestra.run.smithi042:mon.c> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T14:15:20.329 DEBUG:teuthology.orchestra.run.smithi159:mon.b> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T14:15:20.331 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:15:20.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:15:20.988 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi159:172.21.15.159=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: Saving service mon spec with placement smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi159:172.21.15.159=b;count:3 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: Deploying daemon mon.b on smithi159 2024-09-13T14:15:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:20 smithi042 ceph-mon[29349]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:23 smithi042 ceph-mon[29349]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:24.292 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:15:24.292 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":1,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","modified":"2024-09-13T14:13:51.959909Z","created":"2024-09-13T14:13:51.959909Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T14:15:24.292 INFO:teuthology.orchestra.run.smithi159.stderr:dumped monmap epoch 1 2024-09-13T14:15:25.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:25 smithi159 systemd[1]: Started Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:15:25.791 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:15:25.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:15:25.996 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: mon.a calling monitor election 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: mon.b calling monitor election 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: monmap e2: 2 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: fsmap 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: mgrmap e13: y(active, since 39s) 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: overall HEALTH_OK 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:30.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:30.453 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:30.453 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.453 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: mon.a calling monitor election 2024-09-13T14:15:30.454 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.454 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:30.454 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.454 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: mon.b calling monitor election 2024-09-13T14:15:30.454 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: monmap e2: 2 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:30.455 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: fsmap 2024-09-13T14:15:30.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:30.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: mgrmap e13: y(active, since 39s) 2024-09-13T14:15:30.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:15:30.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.457 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:30.457 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:30.457 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:30.829 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:15:30.829 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":2,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","modified":"2024-09-13T14:15:25.003081Z","created":"2024-09-13T14:13:51.959909Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T14:15:30.829 INFO:teuthology.orchestra.run.smithi159.stderr:dumped monmap epoch 2 2024-09-13T14:15:31.109 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:31 smithi159 ceph-mon[33330]: Deploying daemon mon.c on smithi042 2024-09-13T14:15:31.109 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:31 smithi159 ceph-mon[33330]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:31.110 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:31 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/37737751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:31.110 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:31 smithi042 ceph-mon[29349]: Deploying daemon mon.c on smithi042 2024-09-13T14:15:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:31 smithi042 ceph-mon[29349]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:31.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:31 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/37737751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:31.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:32.441 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T14:15:32.442 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T14:15:32.635 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:33.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:32 smithi042 systemd[1]: Starting Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:15:33.325 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[29349]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.325 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 podman[40669]: 2024-09-13 14:15:32.956008325 +0000 UTC m=+0.016041149 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T14:15:33.325 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 podman[40669]: 2024-09-13 14:15:33.062809696 +0000 UTC m=+0.122842516 container create a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T14:15:33.326 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 podman[40669]: 2024-09-13 14:15:33.298530052 +0000 UTC m=+0.358562877 container init a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-2c01458, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:15:33.326 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 podman[40669]: 2024-09-13 14:15:33.30360265 +0000 UTC m=+0.363635472 container start a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-2c01458, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:15:33.326 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:15:33.431 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:33 smithi159 ceph-mon[33330]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable), process ceph-mon, pid 7 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pidfile_write: ignore empty --pid-file 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: load: jerasure load: lrc 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Git sha 0 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Compile date 2024-09-12 19:46:18 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: DB SUMMARY 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: DB Session ID: L0GG90VND1R1C2CYJ7JH 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:15:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:15:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.env: 0x55f3fe799bc0 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.info_log: 0x55f3ff1febe0 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.statistics: (nil) 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.use_fsync: 0 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:15:33.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.db_log_dir: 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.wal_dir: 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:15:33.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.write_buffer_manager: 0x55f3ff20c320 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:15:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.unordered_write: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.row_cache: None 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.wal_filter: None 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:15:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.wal_compression: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:15:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:15:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:15:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:15:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_open_files: -1 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:15:33.608 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Compression algorithms supported: 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kZSTD supported: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:15:33.609 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.merge_operator: 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_filter: None 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:15:33.610 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f3ff1fe7e0) 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_type: 0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: data_block_index_type: 0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_shortening: 1 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: checksum: 4 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: no_block_cache: 0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache: 0x55f3ff1f31f0 2024-09-13T14:15:33.611 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_options: 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: capacity : 536870912 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: num_shard_bits : 4 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: persistent_cache: (nil) 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_size: 4096 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_size_deviation: 10 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_restart_interval: 16 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: metadata_block_size: 4096 2024-09-13T14:15:33.612 INFO:journalctl@ceph.mon.c.smithi042.stdout: partition_filters: 0 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: use_delta_encoding: 1 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: filter_policy: bloomfilter 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: whole_key_filtering: 1 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: verify_compression: 0 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: format_version: 5 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: enable_index_compression: 1 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_align: 0 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: max_auto_readahead_size: 262144 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: prepopulate_block_cache: 0 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:15:33.613 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression: NoCompression 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.num_levels: 7 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:15:33.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:15:33.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:15:33.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:15:33.617 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:15:33.618 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.ttl: 2592000 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:15:33.619 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7d0584dd-e794-46ef-9eb4-e5f00afbee91 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236933330912, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T14:15:33.620 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236933331291, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726236933, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7d0584dd-e794-46ef-9eb4-e5f00afbee91", "db_session_id": "L0GG90VND1R1C2CYJ7JH", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726236933331386, "job": 1, "event": "recovery_finished"} 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f3ff21ee00 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: DB pointer 0x55f3ff312000 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** DB Stats ** 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:15:33.621 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:15:33.622 INFO:journalctl@ceph.mon.c.smithi042.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.6 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:15:33.623 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval compaction: 0.00 GB write, 0.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Block cache BinnedLRUCache@0x55f3ff1f31f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-13T14:15:33.624 INFO:journalctl@ceph.mon.c.smithi042.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: using public_addrv [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: starting mon.c rank -1 at public addrs [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] at bind addrs [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(???) e0 preinit fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).mds e1 new map 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: e1 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:15:33.625 INFO:journalctl@ceph.mon.c.smithi042.stdout: legacy client fscid: -1 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout: No filesystems configured 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:15:33.626 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi159", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:33.627 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:15:33.628 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/640632234' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/640632234' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi159:172.21.15.159=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Saving service mon spec with placement smithi042:172.21.15.42=a;smithi042:[v2:172.21.15.42:3301,v1:172.21.15.42:6790]=c;smithi159:172.21.15.159=b;count:3 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.629 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Deploying daemon mon.b on smithi159 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.a calling monitor election 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.630 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.b calling monitor election 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: monmap e2: 2 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: fsmap 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mgrmap e13: y(active, since 39s) 2024-09-13T14:15:33.631 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: overall HEALTH_OK 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: Deploying daemon mon.c on smithi042 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/37737751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 ceph-mon[40707]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 bash[40669]: a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf 2024-09-13T14:15:33.632 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:33 smithi042 systemd[1]: Started Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: mon.a calling monitor election 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: mon.b calling monitor election 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: mon.c calling monitor election 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: fsmap 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: mgrmap e13: y(active, since 48s) 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:15:38.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:38 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.793 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:15:38.793 INFO:teuthology.orchestra.run.smithi159.stdout:{"epoch":3,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","modified":"2024-09-13T14:15:33.354690Z","created":"2024-09-13T14:13:51.959909Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:3300","nonce":0},{"type":"v1","addr":"172.21.15.159:6789","nonce":0}]},"addr":"172.21.15.159:6789/0","public_addr":"172.21.15.159:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3301","nonce":0},{"type":"v1","addr":"172.21.15.42:6790","nonce":0}]},"addr":"172.21.15.42:6790/0","public_addr":"172.21.15.42:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-13T14:15:38.794 INFO:teuthology.orchestra.run.smithi159.stderr:dumped monmap epoch 3 2024-09-13T14:15:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: mon.a calling monitor election 2024-09-13T14:15:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: mon.b calling monitor election 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: mon.c calling monitor election 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: fsmap 2024-09-13T14:15:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: mgrmap e13: y(active, since 48s) 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: overall HEALTH_OK 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: mon.a calling monitor election 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: mon.b calling monitor election 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: mon.c calling monitor election 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: monmap e3: 3 mons at {a=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],b=[v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0],c=[v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: fsmap 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T14:15:38.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: mgrmap e13: y(active, since 48s) 2024-09-13T14:15:38.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: overall HEALTH_OK 2024-09-13T14:15:38.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:38.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:38 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:39.392 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T14:15:39.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T14:15:39.607 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:39.642 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.642 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/2781755537' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.642 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:39.642 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.643 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/2781755537' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.643 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:39.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:39 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/2781755537' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T14:15:39.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:15:40.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:40.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:40.824 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:40 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:40 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.561 INFO:teuthology.orchestra.run.smithi042.stdout:# minimal ceph.conf for 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:15:41.561 INFO:teuthology.orchestra.run.smithi042.stdout:[global] 2024-09-13T14:15:41.561 INFO:teuthology.orchestra.run.smithi042.stdout: fsid = 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:15:41.561 INFO:teuthology.orchestra.run.smithi042.stdout: mon_host = [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4102957049' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.646 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:41.647 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:15:41.648 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:41 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4102957049' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: Reconfiguring mon.a (unknown last config time)... 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:15:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:41 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4102957049' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:42.232 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T14:15:42.233 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:15:42.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:15:42.264 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:15:42.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:42.333 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:15:42.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T14:15:42.363 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:15:42.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:15:42.430 INFO:tasks.cephadm:Adding mgr.y on smithi042 2024-09-13T14:15:42.431 INFO:tasks.cephadm:Adding mgr.x on smithi159 2024-09-13T14:15:42.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi042=y;smithi159=x' 2024-09-13T14:15:42.660 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:15:42.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:42 smithi159 ceph-mon[33330]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:42.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:42 smithi042 ceph-mon[29349]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:42 smithi042 ceph-mon[40707]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:44.016 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.016 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.016 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:15:44.016 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:44.017 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:44.017 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:44.017 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:43 smithi159 ceph-mon[33330]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:15:44.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[29349]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:44.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:44.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:43 smithi042 ceph-mon[40707]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:15:44.409 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled mgr update... 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:44.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:44.997 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.016 DEBUG:teuthology.orchestra.run.smithi159:mgr.x> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T14:15:45.018 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T14:15:45.019 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T14:15:45.019 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:15:45.019 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:15:45.039 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:15:45.039 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_1 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 14:15:32.779751743 +0000 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.099 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.099 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:15:45.167 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T14:15:45.167 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T14:15:45.167 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000127028 s, 4.0 MB/s 2024-09-13T14:15:45.168 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:15:45.228 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_2 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 962 Links: 1 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 14:15:32.780751735 +0000 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.287 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:45.354 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T14:15:45.354 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T14:15:45.354 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000117127 s, 4.4 MB/s 2024-09-13T14:15:45.356 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:15:45.415 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_3 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 980 Links: 1 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 14:15:32.780751735 +0000 2024-09-13T14:15:45.473 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.474 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.474 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 14:10:44.615075959 +0000 2024-09-13T14:15:45.474 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:15:45.539 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T14:15:45.539 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T14:15:45.540 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000121959 s, 4.2 MB/s 2024-09-13T14:15:45.541 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:15:45.599 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_4 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 964 Links: 1 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 14:15:32.780751735 +0000 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.656 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 14:10:44.614075967 +0000 2024-09-13T14:15:45.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:15:45.722 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T14:15:45.722 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T14:15:45.722 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000113798 s, 4.5 MB/s 2024-09-13T14:15:45.723 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:15:45.781 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:15:45.782 DEBUG:teuthology.orchestra.run.smithi159:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T14:15:45.798 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T14:15:45.798 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_1 2024-09-13T14:15:45.855 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T14:15:45.855 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:45.855 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 968 Links: 1 2024-09-13T14:15:45.855 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:45.855 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:45.856 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-09-13 14:15:23.515847287 +0000 2024-09-13T14:15:45.856 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:45.856 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:45.856 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:45.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T14:15:45.920 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-09-13T14:15:45.920 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-09-13T14:15:45.920 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000190719 s, 2.7 MB/s 2024-09-13T14:15:45.922 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T14:15:45.978 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_2 2024-09-13T14:15:46.010 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:45 smithi159 ceph-mon[33330]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:15:46.010 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:45 smithi159 ceph-mon[33330]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:15:46.010 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:45 smithi159 ceph-mon[33330]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi159=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:46.010 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:45 smithi159 ceph-mon[33330]: Saving service mgr spec with placement smithi042=y;smithi159=x;count:2 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 961 Links: 1 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-09-13 14:15:23.516847267 +0000 2024-09-13T14:15:46.026 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-09-13 14:10:43.857329429 +0000 2024-09-13T14:15:46.027 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-09-13 14:10:43.857329429 +0000 2024-09-13T14:15:46.027 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-09-13 14:10:43.857329429 +0000 2024-09-13T14:15:46.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T14:15:46.090 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-09-13T14:15:46.090 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-09-13T14:15:46.090 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000218967 s, 2.3 MB/s 2024-09-13T14:15:46.091 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T14:15:46.147 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_3 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-09-13 14:15:23.516847267 +0000 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:46.203 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:46.204 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-09-13 14:10:43.858329453 +0000 2024-09-13T14:15:46.204 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T14:15:46.268 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-09-13T14:15:46.269 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-09-13T14:15:46.269 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000199468 s, 2.6 MB/s 2024-09-13T14:15:46.270 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T14:15:46.326 DEBUG:teuthology.orchestra.run.smithi159:> stat /dev/vg_nvme/lv_4 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[29349]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[29349]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[29349]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi159=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[29349]: Saving service mgr spec with placement smithi042=y;smithi159=x;count:2 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[40707]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:15:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[40707]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:15:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[40707]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi042=y;smithi159=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:45 smithi042 ceph-mon[40707]: Saving service mgr spec with placement smithi042=y;smithi159=x;count:2 2024-09-13T14:15:46.384 INFO:teuthology.orchestra.run.smithi159.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Access: 2024-09-13 14:15:23.517847248 +0000 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Modify: 2024-09-13 14:10:43.859329476 +0000 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout:Change: 2024-09-13 14:10:43.859329476 +0000 2024-09-13T14:15:46.385 INFO:teuthology.orchestra.run.smithi159.stdout: Birth: 2024-09-13 14:10:43.859329476 +0000 2024-09-13T14:15:46.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T14:15:46.448 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records in 2024-09-13T14:15:46.448 INFO:teuthology.orchestra.run.smithi159.stderr:1+0 records out 2024-09-13T14:15:46.448 INFO:teuthology.orchestra.run.smithi159.stderr:512 bytes copied, 0.000182092 s, 2.8 MB/s 2024-09-13T14:15:46.449 DEBUG:teuthology.orchestra.run.smithi159:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T14:15:46.505 INFO:tasks.cephadm:Deploying osd.0 on smithi042 with /dev/vg_nvme/lv_4... 2024-09-13T14:15:46.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:15:46.716 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: Deploying daemon mgr.x on smithi159 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[29349]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: Deploying daemon mgr.x on smithi159 2024-09-13T14:15:47.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:47 smithi042 ceph-mon[40707]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:47.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: Deploying daemon mgr.x on smithi159 2024-09-13T14:15:47.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:47 smithi159 ceph-mon[33330]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:47.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:47 smithi159 systemd[1]: Starting Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:15:48.339 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 podman[35366]: 2024-09-13 14:15:47.965391401 +0000 UTC m=+0.015296727 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T14:15:48.339 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 podman[35366]: 2024-09-13 14:15:48.110830941 +0000 UTC m=+0.160736269 container create 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458) 2024-09-13T14:15:48.339 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 podman[35366]: 2024-09-13 14:15:48.33829205 +0000 UTC m=+0.388197376 container init 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:15:48.680 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 podman[35366]: 2024-09-13 14:15:48.342846032 +0000 UTC m=+0.392751359 container start 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=reef-2c01458, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-mgr[35405]: -- 172.21.15.159:0/113035590 <== mon.1 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559422902d00 con 0x5594228d6800 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 bash[35366]: 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 systemd[1]: Started Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:48.464+0000 7efc86a60040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:48.522+0000 7efc86a60040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:15:48.727 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:48.678+0000 7efc86a60040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:15:48.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:48.750+0000 7efc86a60040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:15:48.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:48 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:48.913+0000 7efc86a60040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:15:49.150 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:15:49.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_4 2024-09-13T14:15:49.373 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:15:49.384 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:49.051+0000 7efc86a60040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:15:49.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:49.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.385 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.385 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:49.494 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:49.494 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.494 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:49.495 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:49.637 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:49.383+0000 7efc86a60040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:15:49.637 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:49.444+0000 7efc86a60040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:15:49.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:49 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:49.636+0000 7efc86a60040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:15:50.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.667 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.668 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:50.669 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:50.505+0000 7efc86a60040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:15:50.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:50.571+0000 7efc86a60040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: Reconfiguring mgr.y (unknown last config time)... 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:51.541 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.285+0000 7efc86a60040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:15:51.541 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.400+0000 7efc86a60040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:15:51.541 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.540+0000 7efc86a60040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.601+0000 7efc86a60040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.747+0000 7efc86a60040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:15:51.816 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.815+0000 7efc86a60040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:15:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:15:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:15:51.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:51 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:52.094 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:51.875+0000 7efc86a60040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:15:52.094 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.030+0000 7efc86a60040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:15:52.094 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.093+0000 7efc86a60040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:15:52.363 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.159+0000 7efc86a60040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:15:52.363 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.243+0000 7efc86a60040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:15:52.363 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.303+0000 7efc86a60040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:15:52.363 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:52 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:52.362+0000 7efc86a60040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:15:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:15:53.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: Standby manager daemon x started 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:15:53.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:15:53.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:15:53.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:53 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:15:53.498 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:15:53.134+0000 7efc86a60040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:15:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:53.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:15:53.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:15:53.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:15:53.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:15:53.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:53 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2335561428' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:15:54.335 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:15:54.335 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:15:54.335 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]: dispatch 2024-09-13T14:15:54.335 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]': finished 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]: dispatch 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]': finished 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:15:54.336 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:54 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:15:54.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: mgrmap e14: y(active, since 63s), standbys: x 2024-09-13T14:15:54.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:15:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]: dispatch 2024-09-13T14:15:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/196016072' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0a40fd71-cbc3-4195-b082-c4f19f7f1dd4"}]': finished 2024-09-13T14:15:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: osdmap e5: 1 total, 0 up, 1 in 2024-09-13T14:15:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:54 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:15:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:55 smithi159 ceph-mon[33330]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:55 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2450383370' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:55 smithi042 ceph-mon[29349]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:55 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2450383370' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:55 smithi042 ceph-mon[40707]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:55 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2450383370' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:15:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:57 smithi159 ceph-mon[33330]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:57.551 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:57 smithi042 ceph-mon[29349]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:57.552 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:57 smithi042 ceph-mon[40707]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:59.483 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:15:59 smithi042 ceph-mon[29349]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:59.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:15:59 smithi042 ceph-mon[40707]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:15:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:15:59 smithi159 ceph-mon[33330]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:00.378 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:16:00.378 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:00.378 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:16:00.378 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:00.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T14:16:00.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:01.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:01 smithi042 ceph-mon[29349]: Deploying daemon osd.0 on smithi042 2024-09-13T14:16:01.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:01 smithi042 ceph-mon[29349]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:01.418 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:01 smithi042 ceph-mon[40707]: Deploying daemon osd.0 on smithi042 2024-09-13T14:16:01.418 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:01 smithi042 ceph-mon[40707]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:01.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:01 smithi159 ceph-mon[33330]: Deploying daemon osd.0 on smithi042 2024-09-13T14:16:01.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:01 smithi159 ceph-mon[33330]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:03.325 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:03 smithi042 ceph-mon[29349]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:03.326 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:03 smithi042 ceph-mon[40707]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:03.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:03 smithi159 ceph-mon[33330]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[29349]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[40707]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:05.386 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:05.387 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:05.387 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:05.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:05 smithi159 ceph-mon[33330]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:05.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:05.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:05.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:07.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:07 smithi042 ceph-mon[40707]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:07.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:07 smithi042 ceph-mon[29349]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:07.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:07 smithi159 ceph-mon[33330]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:07.763 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 0 on host 'smithi042' 2024-09-13T14:16:08.400 DEBUG:teuthology.orchestra.run.smithi042:osd.0> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T14:16:08.403 INFO:tasks.cephadm:Deploying osd.1 on smithi042 with /dev/vg_nvme/lv_3... 2024-09-13T14:16:08.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:16:08.646 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:16:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T14:16:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[29349]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:16:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[29349]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:16:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[29349]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:09.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[40707]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:16:09.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[40707]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:16:09.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:09.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-mon[40707]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:09.849 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[44979]: 2024-09-13T14:16:09.774+0000 7f97a2e36640 -1 osd.0 0 waiting for initial osdmap 2024-09-13T14:16:09.849 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:16:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[44979]: 2024-09-13T14:16:09.777+0000 7f979ec5f640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:16:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:09 smithi159 ceph-mon[33330]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:16:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:09 smithi159 ceph-mon[33330]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T14:16:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:09 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:09 smithi159 ceph-mon[33330]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[29349]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[29349]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:16:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[29349]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[40707]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[40707]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:16:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:10 smithi042 ceph-mon[40707]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:11.127 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:10 smithi159 ceph-mon[33330]: from='osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:11.127 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:10 smithi159 ceph-mon[33330]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T14:16:11.127 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:10 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.127 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:10 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:11.127 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:10 smithi159 ceph-mon[33330]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T14:16:11.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:11 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:16:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:11 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:16:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:11 smithi159 ceph-mon[33330]: osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203] boot 2024-09-13T14:16:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:11 smithi159 ceph-mon[33330]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:16:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:11 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:16:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:16:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[29349]: osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203] boot 2024-09-13T14:16:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[29349]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:16:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:16:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:16:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[40707]: osd.0 [v2:172.21.15.42:6802/917582203,v1:172.21.15.42:6803/917582203] boot 2024-09-13T14:16:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[40707]: osdmap e8: 1 total, 1 up, 1 in 2024-09-13T14:16:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:11 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:16:12.219 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:16:12.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_3 2024-09-13T14:16:12.828 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:13.087 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:12 smithi042 ceph-mon[29349]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:16:13.087 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:12 smithi042 ceph-mon[29349]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:13.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:12 smithi042 ceph-mon[40707]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:16:13.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:12 smithi042 ceph-mon[40707]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:13.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:12 smithi159 ceph-mon[33330]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T14:16:13.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:12 smithi159 ceph-mon[33330]: pgmap v40: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:15.476 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:15.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.479 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.479 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.479 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:15.479 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:15.479 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: pgmap v41: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:15.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:15.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:16.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi042 2024-09-13T14:16:16.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi042 to 16068M 2024-09-13T14:16:16.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[29349]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:16.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi042 2024-09-13T14:16:16.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi042 to 16068M 2024-09-13T14:16:16.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:16 smithi042 ceph-mon[40707]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:16.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:16 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi042 2024-09-13T14:16:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:16 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi042 to 16068M 2024-09-13T14:16:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:16 smithi159 ceph-mon[33330]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:17.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:17 smithi159 ceph-mon[33330]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:17 smithi042 ceph-mon[40707]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:17 smithi042 ceph-mon[29349]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1520009479' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]': finished 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4077269351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[29349]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1520009479' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]': finished 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:16:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:18.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4077269351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:18.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:18 smithi042 ceph-mon[40707]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:18.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1520009479' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]: dispatch 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "50b16819-9756-4dd5-957c-a2968d7f22bb"}]': finished 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: osdmap e10: 2 total, 1 up, 2 in 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4077269351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:18 smithi159 ceph-mon[33330]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:21.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:21 smithi042 ceph-mon[29349]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:21.485 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:21 smithi042 ceph-mon[40707]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:21 smithi159 ceph-mon[33330]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:22.475 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:22 smithi042 ceph-mon[29349]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:22.475 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:22 smithi042 ceph-mon[40707]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:22.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:22 smithi159 ceph-mon[33330]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:23.822 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:23 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:16:23.822 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:23 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:23 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:16:23.823 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:23 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:23 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T14:16:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:23 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:24.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:24 smithi042 ceph-mon[29349]: Deploying daemon osd.1 on smithi042 2024-09-13T14:16:24.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:24 smithi042 ceph-mon[29349]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:24 smithi042 ceph-mon[40707]: Deploying daemon osd.1 on smithi042 2024-09-13T14:16:24.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:24 smithi042 ceph-mon[40707]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:24 smithi159 ceph-mon[33330]: Deploying daemon osd.1 on smithi042 2024-09-13T14:16:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:24 smithi159 ceph-mon[33330]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:27.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:27 smithi042 ceph-mon[29349]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:27.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:27 smithi042 ceph-mon[40707]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:27 smithi159 ceph-mon[33330]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:28.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:28 smithi042 ceph-mon[29349]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:28.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:28 smithi042 ceph-mon[40707]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:28.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:28 smithi159 ceph-mon[33330]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:29.537 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:29.537 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:29.537 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:29.538 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:29.538 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:29.538 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:29.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:30 smithi042 ceph-mon[29349]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:30.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:30 smithi042 ceph-mon[40707]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:30.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:30 smithi159 ceph-mon[33330]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:31.488 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 1 on host 'smithi042' 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:16:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.138 DEBUG:teuthology.orchestra.run.smithi042:osd.1> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T14:16:32.141 INFO:tasks.cephadm:Deploying osd.2 on smithi042 with /dev/vg_nvme/lv_2... 2024-09-13T14:16:32.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:32.966 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[29349]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:16:32.966 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[29349]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:16:32.966 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:32.966 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[29349]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:32.966 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[29349]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:32.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[40707]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:16:32.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[40707]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:16:32.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:32.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[40707]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:32.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-mon[40707]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:32.967 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[50965]: 2024-09-13T14:16:32.835+0000 7fa65fb60640 -1 osd.1 0 waiting for initial osdmap 2024-09-13T14:16:32.967 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:16:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[50965]: 2024-09-13T14:16:32.838+0000 7fa65b989640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:16:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:32 smithi159 ceph-mon[33330]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:16:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:32 smithi159 ceph-mon[33330]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T14:16:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:32 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:32 smithi159 ceph-mon[33330]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:32 smithi159 ceph-mon[33330]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:33.766 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[29349]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[29349]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:16:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[40707]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[40707]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:16:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:33 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:33 smithi159 ceph-mon[33330]: from='osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:33 smithi159 ceph-mon[33330]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T14:16:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:33 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:33 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807] boot 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:35.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[29349]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807] boot 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:35.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:34 smithi042 ceph-mon[40707]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: osd.1 [v2:172.21.15.42:6810/3832896807,v1:172.21.15.42:6811/3832896807] boot 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: osdmap e13: 2 total, 2 up, 2 in 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:16:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:34 smithi159 ceph-mon[33330]: pgmap v55: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-13T14:16:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:16:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:36.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:35 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:35 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:36.758 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:16:36.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_2 2024-09-13T14:16:36.985 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi042 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi042 to 8034M 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[29349]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi042 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi042 to 8034M 2024-09-13T14:16:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:36 smithi042 ceph-mon[40707]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:36 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi042 2024-09-13T14:16:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:36 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi042 to 8034M 2024-09-13T14:16:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:36 smithi159 ceph-mon[33330]: pgmap v57: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:39.392 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[29349]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[40707]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:39.393 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:39 smithi159 ceph-mon[33330]: pgmap v58: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:16:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:16:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:40.426 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:40 smithi042 ceph-mon[29349]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.426 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:40 smithi042 ceph-mon[40707]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:40 smithi159 ceph-mon[33330]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:16:41.485 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1139859426' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]': finished 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1139859426' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]': finished 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:16:41.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:41 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1139859426' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]: dispatch 2024-09-13T14:16:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "de5587ea-c587-4c38-a8ce-6f2bf7f41b2d"}]': finished 2024-09-13T14:16:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: osdmap e15: 3 total, 2 up, 3 in 2024-09-13T14:16:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:41 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:42.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:42 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2309932035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:42 smithi159 ceph-mon[33330]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:42 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2309932035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:42 smithi042 ceph-mon[40707]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:42 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2309932035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:16:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:42 smithi042 ceph-mon[29349]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:45.392 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:45 smithi042 ceph-mon[29349]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:45.392 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:45 smithi042 ceph-mon[40707]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:45 smithi159 ceph-mon[33330]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:46.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:46 smithi159 ceph-mon[33330]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:46.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:46 smithi042 ceph-mon[40707]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:46.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:46 smithi042 ceph-mon[29349]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:47.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:16:47.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:16:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:47.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T14:16:47.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:48 smithi042 ceph-mon[40707]: Deploying daemon osd.2 on smithi042 2024-09-13T14:16:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:48 smithi042 ceph-mon[40707]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:48 smithi042 ceph-mon[29349]: Deploying daemon osd.2 on smithi042 2024-09-13T14:16:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:48 smithi042 ceph-mon[29349]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:48.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:48 smithi159 ceph-mon[33330]: Deploying daemon osd.2 on smithi042 2024-09-13T14:16:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:48 smithi159 ceph-mon[33330]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:51.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:51 smithi159 ceph-mon[33330]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:51 smithi042 ceph-mon[29349]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:51 smithi042 ceph-mon[40707]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[40707]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[29349]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:52 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:52 smithi159 ceph-mon[33330]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:52 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:52 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:52 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.423 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 2 on host 'smithi042' 2024-09-13T14:16:55.423 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:55.425 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:55.425 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:16:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.027 DEBUG:teuthology.orchestra.run.smithi042:osd.2> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T14:16:56.029 INFO:tasks.cephadm:Deploying osd.3 on smithi042 with /dev/vg_nvme/lv_1... 2024-09-13T14:16:56.029 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:16:56.258 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.258 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[40707]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.259 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:56 smithi042 ceph-mon[29349]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:56.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T14:16:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:16:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:16:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:56 smithi159 ceph-mon[33330]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:57.226 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:16:57.237 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[29349]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:16:57.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[29349]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:16:57.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[29349]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[29349]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[40707]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[40707]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[40707]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-mon[40707]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:57 smithi159 ceph-mon[33330]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:16:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:57 smithi159 ceph-mon[33330]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T14:16:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:57 smithi159 ceph-mon[33330]: from='osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:57 smithi159 ceph-mon[33330]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:16:57.743 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[56893]: 2024-09-13T14:16:57.499+0000 7f1a5c3e9640 -1 osd.2 0 waiting for initial osdmap 2024-09-13T14:16:57.743 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:16:57 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[56893]: 2024-09-13T14:16:57.503+0000 7f1a5420a640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: from='osd.2 ' entity='osd.2' 2024-09-13T14:16:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[40707]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: from='osd.2 ' entity='osd.2' 2024-09-13T14:16:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:58 smithi042 ceph-mon[29349]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:58.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:16:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T14:16:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: from='osd.2 ' entity='osd.2' 2024-09-13T14:16:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:58 smithi159 ceph-mon[33330]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888] boot 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:16:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888] boot 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:16:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:16:59 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:16:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:16:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:16:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: osd.2 [v2:172.21.15.42:6818/2844866888,v1:172.21.15.42:6819/2844866888] boot 2024-09-13T14:16:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: osdmap e18: 3 total, 3 up, 3 in 2024-09-13T14:16:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:16:59 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:00.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:00.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:00 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.827 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:17:00.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_1 2024-09-13T14:17:00.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:00 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:01.046 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:17:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi042 2024-09-13T14:17:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi042 to 5356M 2024-09-13T14:17:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: pgmap v73: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi042 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi042 to 5356M 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: pgmap v73: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:17:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:01 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:01.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi042 2024-09-13T14:17:01.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi042 to 5356M 2024-09-13T14:17:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: pgmap v73: 0 pgs: ; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T14:17:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:01 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:17:02.436 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61249]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:02.436 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61249]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:02.436 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61249]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:02.436 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61249]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[29349]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:17:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[29349]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:02.848 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61307]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:02.848 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61307]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:02.848 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61307]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:02.849 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:17:02 smithi042 sudo[61307]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[40707]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:17:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:02 smithi042 ceph-mon[40707]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:02.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:02 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:17:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:02 smithi159 ceph-mon[33330]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T14:17:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:02 smithi159 ceph-mon[33330]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:03.469 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61431]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:03.470 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61431]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:03.470 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61431]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:03.470 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61431]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:03.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61471]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:03.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61471]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:03.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61471]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:03.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:17:03 smithi042 sudo[61471]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:03.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:03.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:04.215 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61483]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:04.215 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61483]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:04.215 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61483]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:04.215 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61483]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:04.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61521]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:04.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61521]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:04.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61521]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:04.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61521]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61558]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61558]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61558]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:04 smithi042 sudo[61558]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:05 smithi042 ceph-mon[40707]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:05 smithi042 sudo[61581]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:05 smithi042 sudo[61581]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:05 smithi042 sudo[61581]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:05 smithi042 sudo[61581]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:05.663 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:05 smithi042 ceph-mon[29349]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:05.671 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 ceph-mon[33330]: pgmap v78: 1 pgs: 1 unknown; 0 B data, 84 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:05.671 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36053]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:05.671 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36053]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:05.672 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36053]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:05.672 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36053]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:05.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36056]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:05.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36056]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36056]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:05 smithi159 sudo[36056]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:06.832 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61665]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T14:17:06.832 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61665]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:06.832 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61665]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:06.832 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61665]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61852]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T14:17:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61852]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-13T14:17:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61852]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 sudo[61852]: pam_unix(sudo:session): session closed for user root 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1721894789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]': finished 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/504002204' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[40707]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1721894789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]': finished 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/504002204' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:07.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:06 smithi042 ceph-mon[29349]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1721894789' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0944a1a9-ea18-4457-8d5e-cf70f7e2be0f"}]': finished 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: osdmap e22: 4 total, 3 up, 4 in 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/504002204' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:07.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:06 smithi159 ceph-mon[33330]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 85 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:07 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:07 smithi042 ceph-mon[29349]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:07 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:07 smithi042 ceph-mon[40707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:07 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T14:17:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:07 smithi159 ceph-mon[33330]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T14:17:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:08 smithi042 ceph-mon[29349]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:08 smithi042 ceph-mon[40707]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:08 smithi159 ceph-mon[33330]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:10.590 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:10 smithi042 ceph-mon[40707]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 165 KiB/s wr, 11 op/s 2024-09-13T14:17:10.590 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:10 smithi042 ceph-mon[29349]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 165 KiB/s wr, 11 op/s 2024-09-13T14:17:10.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:10 smithi159 ceph-mon[33330]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 3.8 KiB/s rd, 165 KiB/s wr, 11 op/s 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[29349]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 116 KiB/s wr, 8 op/s 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[40707]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 116 KiB/s wr, 8 op/s 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:17:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:12.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:12 smithi159 ceph-mon[33330]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 116 KiB/s wr, 8 op/s 2024-09-13T14:17:12.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T14:17:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:13.277 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:13 smithi042 ceph-mon[29349]: Deploying daemon osd.3 on smithi042 2024-09-13T14:17:13.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:13 smithi042 ceph-mon[40707]: Deploying daemon osd.3 on smithi042 2024-09-13T14:17:13.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:13 smithi159 ceph-mon[33330]: Deploying daemon osd.3 on smithi042 2024-09-13T14:17:14.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:14 smithi042 ceph-mon[29349]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 89 KiB/s wr, 6 op/s 2024-09-13T14:17:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:14 smithi042 ceph-mon[40707]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 89 KiB/s wr, 6 op/s 2024-09-13T14:17:14.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:14 smithi159 ceph-mon[33330]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 2.1 KiB/s rd, 89 KiB/s wr, 6 op/s 2024-09-13T14:17:16.502 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:16 smithi042 ceph-mon[29349]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T14:17:16.502 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:16 smithi042 ceph-mon[40707]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T14:17:16.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:16 smithi159 ceph-mon[33330]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 73 KiB/s wr, 5 op/s 2024-09-13T14:17:17.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:17 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:17.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:17 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:17 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:17.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:17.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:17.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:17.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:17.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:17.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:17 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:18 smithi042 ceph-mon[29349]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:18.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:18 smithi042 ceph-mon[40707]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:18.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:18 smithi159 ceph-mon[33330]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:20.706 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 3 on host 'smithi042' 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T14:17:21.304 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:21.304 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.304 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:21.324 DEBUG:teuthology.orchestra.run.smithi042:osd.3> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T14:17:21.326 INFO:tasks.cephadm:Deploying osd.4 on smithi159 with /dev/vg_nvme/lv_4... 2024-09-13T14:17:21.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T14:17:21.530 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:22.458 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[29349]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[29349]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[29349]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[29349]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[40707]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[40707]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[40707]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:17:22.459 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-mon[40707]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:22.460 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[63377]: 2024-09-13T14:17:22.375+0000 7f5d762d6640 -1 osd.3 0 waiting for initial osdmap 2024-09-13T14:17:22.460 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:17:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[63377]: 2024-09-13T14:17:22.380+0000 7f5d720ff640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:17:22.633 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:22 smithi159 ceph-mon[33330]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:17:22.633 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:22 smithi159 ceph-mon[33330]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T14:17:22.633 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:22 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:22.633 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:22 smithi159 ceph-mon[33330]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T14:17:22.634 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:22 smithi159 ceph-mon[33330]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:23.697 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:23 smithi159 ceph-mon[33330]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:17:23.697 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:23 smithi159 ceph-mon[33330]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:17:23.697 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:23 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.697 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:23 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[29349]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:17:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[29349]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:17:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[40707]: from='osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T14:17:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[40707]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T14:17:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:23 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:23.967 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:17:23.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi159:vg_nvme/lv_4 2024-09-13T14:17:24.181 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:24.578 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:17:24.578 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:17:24.578 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887] boot 2024-09-13T14:17:24.579 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:17:24.579 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:24.579 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:24.579 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi042 2024-09-13T14:17:24.579 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.581 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.581 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:24.581 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:24.581 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:24 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887] boot 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi042 2024-09-13T14:17:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:17:24.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: osd.3 [v2:172.21.15.42:6826/38876887,v1:172.21.15.42:6827/38876887] boot 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: osdmap e25: 4 total, 4 up, 4 in 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 86 MiB used, 268 GiB / 268 GiB avail 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi042 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:24.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:24.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:24.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:24 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:25.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:25 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi042 to 4017M 2024-09-13T14:17:25.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:25 smithi159 ceph-mon[33330]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:17:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:25 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi042 to 4017M 2024-09-13T14:17:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:25 smithi042 ceph-mon[29349]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:17:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:25 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi042 to 4017M 2024-09-13T14:17:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:25 smithi042 ceph-mon[40707]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T14:17:26.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:26 smithi159 ceph-mon[33330]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:26.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:26.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:26.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:26.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:26 smithi159 ceph-mon[33330]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[29349]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[29349]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:26.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[40707]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:26.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:26.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:26.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:26.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:26 smithi042 ceph-mon[40707]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:28.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:28 smithi042 ceph-mon[29349]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:28.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:28 smithi042 ceph-mon[40707]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:28.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:28 smithi159 ceph-mon[33330]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/2893339415' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]': finished 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/97314477' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/2893339415' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]': finished 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:29 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/97314477' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:29.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/2893339415' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]: dispatch 2024-09-13T14:17:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9cb88075-7a94-4890-8de9-a717d57eae24"}]': finished 2024-09-13T14:17:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: osdmap e27: 5 total, 4 up, 5 in 2024-09-13T14:17:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:29 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/97314477' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:30 smithi042 ceph-mon[29349]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:30 smithi042 ceph-mon[40707]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:30.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:30 smithi159 ceph-mon[33330]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:32 smithi042 ceph-mon[29349]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:32.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:32 smithi042 ceph-mon[40707]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:32.637 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:32 smithi159 ceph-mon[33330]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:34.424 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:34 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:34.425 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:34 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:34 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:34 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:34 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T14:17:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:34 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:35.308 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:35 smithi159 ceph-mon[33330]: Deploying daemon osd.4 on smithi159 2024-09-13T14:17:35.308 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:35 smithi159 ceph-mon[33330]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:35 smithi042 ceph-mon[29349]: Deploying daemon osd.4 on smithi159 2024-09-13T14:17:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:35 smithi042 ceph-mon[29349]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:35 smithi042 ceph-mon[40707]: Deploying daemon osd.4 on smithi159 2024-09-13T14:17:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:35 smithi042 ceph-mon[40707]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:36.406 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:36 smithi159 ceph-mon[33330]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:36 smithi042 ceph-mon[29349]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:36.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:36 smithi042 ceph-mon[40707]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:38.322 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:38 smithi159 ceph-mon[33330]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:38.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:38 smithi042 ceph-mon[29349]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:38 smithi042 ceph-mon[40707]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:39.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:39.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:40 smithi159 ceph-mon[33330]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:40 smithi042 ceph-mon[29349]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:40 smithi042 ceph-mon[40707]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:42.003 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 4 on host 'smithi159' 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:42 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:42 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:42.601 DEBUG:teuthology.orchestra.run.smithi159:osd.4> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T14:17:42.603 INFO:tasks.cephadm:Deploying osd.5 on smithi159 with /dev/vg_nvme/lv_3... 2024-09-13T14:17:42.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T14:17:43.157 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:43.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:17:43.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:43.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-mon[33330]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:43.158 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T14:17:43.058+0000 7f9e9d1aa640 -1 osd.4 0 waiting for initial osdmap 2024-09-13T14:17:43.158 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 14:17:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T14:17:43.062+0000 7f9e98fd3640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:17:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:17:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[29349]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: from='osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:17:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:43 smithi042 ceph-mon[40707]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 112 MiB used, 357 GiB / 358 GiB avail 2024-09-13T14:17:43.373 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[29349]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[29349]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[40707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:17:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[40707]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:44 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:44.439 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:44 smithi159 ceph-mon[33330]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:17:44.440 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:44 smithi159 ceph-mon[33330]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T14:17:44.440 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:44.440 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:44 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460] boot 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:45.312 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:45 smithi159 ceph-mon[33330]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:17:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:17:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460] boot 2024-09-13T14:17:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[29349]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: osd.4 [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460] boot 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: osdmap e30: 5 total, 5 up, 5 in 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:17:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:45 smithi042 ceph-mon[40707]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.326 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:46 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:46.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:46 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi159 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 17088M 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:47.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[29349]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi159 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 17088M 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:47 smithi042 ceph-mon[40707]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi159 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 17088M 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:17:47.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:47 smithi159 ceph-mon[33330]: pgmap v110: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:47.384 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:17:47.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi159:vg_nvme/lv_3 2024-09-13T14:17:47.596 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:17:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:48 smithi042 ceph-mon[29349]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:48 smithi042 ceph-mon[40707]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:48.379 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:48 smithi159 ceph-mon[33330]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T14:17:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:49 smithi159 ceph-mon[33330]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T14:17:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:49 smithi159 ceph-mon[33330]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:49.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:49 smithi042 ceph-mon[29349]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T14:17:49.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:49 smithi042 ceph-mon[29349]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:49.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:49 smithi042 ceph-mon[40707]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T14:17:49.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:49 smithi042 ceph-mon[40707]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:17:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:50.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:50.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:50.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:50 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:50.379 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:17:50.379 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:17:50.380 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:50 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:51.182 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:51 smithi159 ceph-mon[33330]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:51.182 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:51 smithi159 ceph-mon[33330]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-13T14:17:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:51 smithi042 ceph-mon[29349]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:51 smithi042 ceph-mon[29349]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-13T14:17:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:51 smithi042 ceph-mon[40707]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:17:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:51 smithi042 ceph-mon[40707]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-09-13T14:17:52.215 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:52 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/3462621102' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.215 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:52 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.215 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:52 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]': finished 2024-09-13T14:17:52.215 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:52 smithi159 ceph-mon[33330]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:17:52.215 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:52 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/3462621102' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]': finished 2024-09-13T14:17:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[29349]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:17:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/3462621102' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]: dispatch 2024-09-13T14:17:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70e09ea8-aa92-41de-9d8b-f15ebee3edac"}]': finished 2024-09-13T14:17:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[40707]: osdmap e35: 6 total, 5 up, 6 in 2024-09-13T14:17:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:52 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:17:53.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:53 smithi042 ceph-mon[29349]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:53.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:53 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/4175382881' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:53 smithi042 ceph-mon[40707]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:53 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/4175382881' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:53.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:53 smithi159 ceph-mon[33330]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:17:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:53 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/4175382881' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:17:54.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:54 smithi042 ceph-mon[29349]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-13T14:17:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:54 smithi042 ceph-mon[40707]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-13T14:17:54.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:54 smithi159 ceph-mon[33330]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-13T14:17:56.444 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:56 smithi159 ceph-mon[33330]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:17:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:56 smithi042 ceph-mon[29349]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:17:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:56 smithi042 ceph-mon[40707]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T14:17:57.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:17:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:17:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T14:17:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:17:59.064 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:58 smithi159 ceph-mon[33330]: Deploying daemon osd.5 on smithi159 2024-09-13T14:17:59.064 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:17:58 smithi159 ceph-mon[33330]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:17:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:58 smithi042 ceph-mon[29349]: Deploying daemon osd.5 on smithi159 2024-09-13T14:17:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:17:58 smithi042 ceph-mon[29349]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:17:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:58 smithi042 ceph-mon[40707]: Deploying daemon osd.5 on smithi159 2024-09-13T14:17:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:17:58 smithi042 ceph-mon[40707]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:18:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:00 smithi159 ceph-mon[33330]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:00 smithi042 ceph-mon[29349]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:00 smithi042 ceph-mon[40707]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:02 smithi159 ceph-mon[33330]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:02 smithi042 ceph-mon[29349]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:02 smithi042 ceph-mon[40707]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:03.461 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:03.461 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:03.461 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:03 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:03 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:04.350 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:04 smithi159 ceph-mon[33330]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:04.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:04 smithi042 ceph-mon[40707]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:04.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:04 smithi042 ceph-mon[29349]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:05.706 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 5 on host 'smithi159' 2024-09-13T14:18:06.325 DEBUG:teuthology.orchestra.run.smithi159:osd.5> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T14:18:06.327 INFO:tasks.cephadm:Deploying osd.6 on smithi159 with /dev/vg_nvme/lv_2... 2024-09-13T14:18:06.327 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T14:18:06.574 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:06.947 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:06.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:06 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:06 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:07 smithi159 ceph-mon[33330]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:07 smithi159 ceph-mon[33330]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:07 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:07 smithi159 ceph-mon[33330]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:07 smithi159 ceph-mon[33330]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[40707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[40707]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[40707]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[40707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T14:18:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[29349]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T14:18:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[29349]: from='osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:07 smithi042 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:08.499 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T14:18:08.131+0000 7fb9ca168640 -1 osd.5 0 waiting for initial osdmap 2024-09-13T14:18:08.499 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T14:18:08.143+0000 7fb9c1f89640 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: from='osd.5 ' entity='osd.5' 2024-09-13T14:18:08.793 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:08 smithi159 ceph-mon[33330]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: from='osd.5 ' entity='osd.5' 2024-09-13T14:18:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[29349]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: from='osd.5 ' entity='osd.5' 2024-09-13T14:18:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:08 smithi042 ceph-mon[40707]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-09-13T14:18:09.817 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:18:09.834 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi159:vg_nvme/lv_2 2024-09-13T14:18:09.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:09 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:18:09.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:09 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:18:09.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:09 smithi159 ceph-mon[33330]: osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230] boot 2024-09-13T14:18:09.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:09 smithi159 ceph-mon[33330]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:09.994 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:09 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:09.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:18:09.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:18:09.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[40707]: osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230] boot 2024-09-13T14:18:09.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[40707]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[29349]: osd.5 [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230] boot 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[29349]: osdmap e38: 6 total, 6 up, 6 in 2024-09-13T14:18:10.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:09 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:18:10.402 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:10.833 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:10 smithi159 ceph-mon[33330]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:10.833 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:10 smithi159 ceph-mon[33330]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:10.833 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:10 smithi159 ceph-mon[33330]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T14:18:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[29349]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[29349]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[29349]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T14:18:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[40707]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T14:18:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[40707]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:10 smithi042 ceph-mon[40707]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi159 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 8544M 2024-09-13T14:18:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:12.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:12 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T14:18:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi159 2024-09-13T14:18:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 8544M 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi159 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:13.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 8544M 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:13.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:12 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:13.912 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:13 smithi159 ceph-mon[33330]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:13.913 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:13 smithi159 ceph-mon[33330]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T14:18:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:13 smithi042 ceph-mon[29349]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:13 smithi042 ceph-mon[29349]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T14:18:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:13 smithi042 ceph-mon[40707]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:13 smithi042 ceph-mon[40707]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T14:18:14.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:14 smithi159 ceph-mon[33330]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:14 smithi042 ceph-mon[29349]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:14 smithi042 ceph-mon[40707]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 165 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/2799057181' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]': finished 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/3737496894' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/2799057181' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]': finished 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:15 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/3737496894' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/2799057181' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]: dispatch 2024-09-13T14:18:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cc9e1f9f-78f5-452a-9f78-a47340fbdefe"}]': finished 2024-09-13T14:18:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: osdmap e43: 7 total, 6 up, 7 in 2024-09-13T14:18:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:15 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/3737496894' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:16 smithi042 ceph-mon[29349]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:16 smithi042 ceph-mon[40707]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:17.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:16 smithi159 ceph-mon[33330]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:18.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:18 smithi159 ceph-mon[33330]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:18 smithi042 ceph-mon[29349]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:18.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:18 smithi042 ceph-mon[40707]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:20 smithi159 ceph-mon[33330]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:20 smithi042 ceph-mon[29349]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:20 smithi042 ceph-mon[40707]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:21 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:21 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:21 smithi159 ceph-mon[33330]: Deploying daemon osd.6 on smithi159 2024-09-13T14:18:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[29349]: Deploying daemon osd.6 on smithi159 2024-09-13T14:18:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T14:18:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:21.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:21 smithi042 ceph-mon[40707]: Deploying daemon osd.6 on smithi159 2024-09-13T14:18:22.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:22 smithi159 ceph-mon[33330]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:22.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:22 smithi042 ceph-mon[29349]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:22 smithi042 ceph-mon[40707]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:24.362 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:24 smithi159 ceph-mon[33330]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:24.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:24 smithi042 ceph-mon[29349]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:24 smithi042 ceph-mon[40707]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:26 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:26 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:27.275 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:27 smithi159 ceph-mon[33330]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:27 smithi042 ceph-mon[29349]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:27.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:27 smithi042 ceph-mon[40707]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:28.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:28 smithi159 ceph-mon[33330]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:28.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:28 smithi042 ceph-mon[29349]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:28.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:28 smithi042 ceph-mon[40707]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:29.136 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 6 on host 'smithi159' 2024-09-13T14:18:29.446 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:29 smithi159 ceph-mon[33330]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.447 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:29 smithi159 ceph-mon[33330]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.447 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.447 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:29 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[29349]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[29349]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[40707]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[40707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T14:18:29.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:29 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:29.751 DEBUG:teuthology.orchestra.run.smithi159:osd.6> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T14:18:29.754 INFO:tasks.cephadm:Deploying osd.7 on smithi159 with /dev/vg_nvme/lv_1... 2024-09-13T14:18:29.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T14:18:29.998 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.401 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-mon[33330]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:30.401 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T14:18:30.304+0000 7f09d6228640 -1 osd.6 0 waiting for initial osdmap 2024-09-13T14:18:30.402 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 14:18:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T14:18:30.310+0000 7f09d2051640 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[40707]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:30 smithi042 ceph-mon[29349]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 166 MiB used, 536 GiB / 536 GiB avail 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602] boot 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T14:18:31.399 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:31 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602] boot 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T14:18:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: osd.6 [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602] boot 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: osdmap e46: 7 total, 7 up, 7 in 2024-09-13T14:18:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:31 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:18:32.372 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:32 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:18:32.373 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:32 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:18:32.373 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:32 smithi159 ceph-mon[33330]: pgmap v147: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:32.373 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:32 smithi159 ceph-mon[33330]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T14:18:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:18:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:18:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[29349]: pgmap v147: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[29349]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T14:18:32.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:18:32.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:18:32.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[40707]: pgmap v147: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:32.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:32 smithi042 ceph-mon[40707]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T14:18:32.785 INFO:teuthology.orchestra.run.smithi159.stdout: 2024-09-13T14:18:32.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi159:vg_nvme/lv_1 2024-09-13T14:18:33.626 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:18:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:34 smithi042 ceph-mon[29349]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T14:18:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:34 smithi042 ceph-mon[29349]: pgmap v150: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:34 smithi042 ceph-mon[40707]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T14:18:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:34 smithi042 ceph-mon[40707]: pgmap v150: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:34.686 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:34 smithi159 ceph-mon[33330]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T14:18:34.686 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:34 smithi159 ceph-mon[33330]: pgmap v150: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:35 smithi042 ceph-mon[29349]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T14:18:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:35 smithi042 ceph-mon[40707]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T14:18:35.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:35 smithi159 ceph-mon[33330]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T14:18:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T14:18:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi159 2024-09-13T14:18:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 5696M 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[29349]: pgmap v153: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi159 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 5696M 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:36 smithi042 ceph-mon[40707]: pgmap v153: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:36.648 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T14:18:36.648 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi159 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 5696M 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.649 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi159:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:36.650 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:36 smithi159 ceph-mon[33330]: pgmap v153: 1 pgs: 1 remapped; 577 KiB data, 193 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:38 smithi159 ceph-mon[33330]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-13T14:18:38.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:38 smithi042 ceph-mon[29349]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-13T14:18:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:38 smithi042 ceph-mon[40707]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-13T14:18:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/1450512521' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]': finished 2024-09-13T14:18:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T14:18:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:39 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/303788226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/1450512521' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]': finished 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/303788226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/1450512521' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]: dispatch 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85448d6e-e256-43f5-aa86-bf8c47b40863"}]': finished 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: osdmap e51: 8 total, 7 up, 8 in 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:39 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/303788226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T14:18:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:40 smithi042 ceph-mon[29349]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:18:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:40 smithi042 ceph-mon[40707]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:18:40.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:40 smithi159 ceph-mon[33330]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-13T14:18:42.535 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:42 smithi159 ceph-mon[33330]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:18:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:42 smithi042 ceph-mon[29349]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:18:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:42 smithi042 ceph-mon[40707]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-13T14:18:44.471 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:44 smithi159 ceph-mon[33330]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:18:44.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:44 smithi042 ceph-mon[29349]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:18:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:44 smithi042 ceph-mon[40707]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T14:18:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:45 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:18:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:45 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:45 smithi159 ceph-mon[33330]: Deploying daemon osd.7 on smithi159 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[29349]: Deploying daemon osd.7 on smithi159 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:45.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:45 smithi042 ceph-mon[40707]: Deploying daemon osd.7 on smithi159 2024-09-13T14:18:46.528 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:46 smithi159 ceph-mon[33330]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:18:46.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:46 smithi042 ceph-mon[29349]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:18:46.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:46 smithi042 ceph-mon[40707]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:18:48.443 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:48 smithi159 ceph-mon[33330]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:48 smithi042 ceph-mon[29349]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:48 smithi042 ceph-mon[40707]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:49.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:49.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:49.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:49 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:49 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:50 smithi159 ceph-mon[33330]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:50 smithi042 ceph-mon[29349]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:50 smithi042 ceph-mon[40707]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:52.879 INFO:teuthology.orchestra.run.smithi159.stdout:Created osd(s) 7 on host 'smithi159' 2024-09-13T14:18:53.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:53.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:53.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.476 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:53 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.495 DEBUG:teuthology.orchestra.run.smithi159:osd.7> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T14:18:53.498 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T14:18:53.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T14:18:53.527 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:53.527 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.527 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.527 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:53 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:53.717 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:18:54.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-mon[33330]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:18:54.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-mon[33330]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T14:18:54.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:54.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-mon[33330]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:54.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-mon[33330]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:54.382 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T14:18:54.215+0000 7f6fd521c640 -1 osd.7 0 waiting for initial osdmap 2024-09-13T14:18:54.382 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 14:18:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T14:18:54.223+0000 7f6fd1045640 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[40707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[40707]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[40707]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[40707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:54.514 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T14:18:54.515 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[29349]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T14:18:54.515 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:54.515 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[29349]: from='osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:54.515 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:54 smithi042 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]: dispatch 2024-09-13T14:18:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:55 smithi159 ceph-mon[33330]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:55 smithi159 ceph-mon[33330]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:55 smithi159 ceph-mon[33330]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T14:18:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:55 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[40707]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[40707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[40707]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T14:18:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[29349]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 194 MiB used, 626 GiB / 626 GiB avail 2024-09-13T14:18:55.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[29349]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi159", "root=default"]}]': finished 2024-09-13T14:18:55.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[29349]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T14:18:55.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:55 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:55.653 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:18:56.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: purged_snaps scrub starts 2024-09-13T14:18:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: purged_snaps scrub ok 2024-09-13T14:18:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602] boot 2024-09-13T14:18:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:18:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:56 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/787735087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:18:56.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: purged_snaps scrub starts 2024-09-13T14:18:56.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: purged_snaps scrub ok 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602] boot 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/787735087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: purged_snaps scrub starts 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: purged_snaps scrub ok 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: osd.7 [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602] boot 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: osdmap e54: 8 total, 8 up, 8 in 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:18:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:56 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/787735087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T14:18:56.623 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":54,"num_osds":8,"num_up_osds":8,"osd_up_since":1726237135,"num_in_osds":8,"osd_in_since":1726237118,"num_remapped_pgs":1} 2024-09-13T14:18:56.623 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:18:56.869 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:57.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:57.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:57 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:18:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:57 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:18:58.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:58 smithi159 ceph-mon[33330]: Detected new or changed devices on smithi159 2024-09-13T14:18:58.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:58 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 4272M 2024-09-13T14:18:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:58 smithi159 ceph-mon[33330]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:18:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:58 smithi159 ceph-mon[33330]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[29349]: Detected new or changed devices on smithi159 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 4272M 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[29349]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[29349]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[40707]: Detected new or changed devices on smithi159 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 4272M 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[40707]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T14:18:58.547 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:58 smithi042 ceph-mon[40707]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T14:18:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:18:58.995 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":58,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:18:58.643618+0000","last_up_change":"2024-09-13T14:18:55.210301+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6803","nonce":917582203}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6805","nonce":917582203}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6809","nonce":917582203}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6807","nonce":917582203}]},"public_addr":"172.21.15.42:6803/917582203","cluster_addr":"172.21.15.42:6805/917582203","heartbeat_back_addr":"172.21.15.42:6809/917582203","heartbeat_front_addr":"172.21.15.42:6807/917582203","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6811","nonce":3832896807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6813","nonce":3832896807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6817","nonce":3832896807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6815","nonce":3832896807}]},"public_addr":"172.21.15.42:6811/3832896807","cluster_addr":"172.21.15.42:6813/3832896807","heartbeat_back_addr":"172.21.15.42:6817/3832896807","heartbeat_front_addr":"172.21.15.42:6815/3832896807","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6819","nonce":2844866888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6821","nonce":2844866888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6825","nonce":2844866888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6823","nonce":2844866888}]},"public_addr":"172.21.15.42:6819/2844866888","cluster_addr":"172.21.15.42:6821/2844866888","heartbeat_back_addr":"172.21.15.42:6825/2844866888","heartbeat_front_addr":"172.21.15.42:6823/2844866888","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6827","nonce":38876887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6829","nonce":38876887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6833","nonce":38876887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6831","nonce":38876887}]},"public_addr":"172.21.15.42:6827/38876887","cluster_addr":"172.21.15.42:6829/38876887","heartbeat_back_addr":"172.21.15.42:6833/38876887","heartbeat_front_addr":"172.21.15.42:6831/38876887","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:18:59.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[29349]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:59.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[29349]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:18:59.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/306192815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.364 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[40707]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:59.364 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[40707]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:18:59.364 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:18:59 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/306192815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:59 smithi159 ceph-mon[33330]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:18:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:59 smithi159 ceph-mon[33330]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T14:18:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:18:59 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/306192815' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:18:59.783 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T14:17:00.645575+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-09-13T14:18:59.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T14:18:59.992 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:01.444 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:01 smithi042 ceph-mon[29349]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:01.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:01 smithi042 ceph-mon[40707]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:01 smithi159 ceph-mon[33330]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:01.835 INFO:teuthology.orchestra.run.smithi042.stdout:pg_num: 1 2024-09-13T14:19:02.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:02 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4053522020' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:19:02.511 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:02 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4053522020' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:19:02.511 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:02 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4053522020' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T14:19:02.539 INFO:tasks.cephadm:Adding prometheus.a on smithi159 2024-09-13T14:19:02.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi159=a' 2024-09-13T14:19:02.735 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:03.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:03 smithi159 ceph-mon[33330]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-13T14:19:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:03 smithi042 ceph-mon[29349]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-13T14:19:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:03 smithi042 ceph-mon[40707]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-13T14:19:04.435 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled prometheus update... 2024-09-13T14:19:05.022 DEBUG:teuthology.orchestra.run.smithi159:prometheus.a> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T14:19:05.024 INFO:tasks.cephadm:Adding node-exporter.a on smithi042 2024-09-13T14:19:05.024 INFO:tasks.cephadm:Adding node-exporter.b on smithi159 2024-09-13T14:19:05.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi042=a;smithi159=b' 2024-09-13T14:19:05.267 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:05.387 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-13T14:19:05.387 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:05.387 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.388 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:05.388 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:05.388 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:05 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:05.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:05 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:06.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:06 smithi159 ceph-mon[33330]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:06 smithi159 ceph-mon[33330]: Saving service prometheus spec with placement smithi159=a;count:1 2024-09-13T14:19:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:06 smithi159 ceph-mon[33330]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[29349]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[29349]: Saving service prometheus spec with placement smithi159=a;count:1 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[29349]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[40707]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[40707]: Saving service prometheus spec with placement smithi159=a;count:1 2024-09-13T14:19:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:06 smithi042 ceph-mon[40707]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:19:07.308 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:07 smithi159 ceph-mon[33330]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:19:07.573 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled node-exporter update... 2024-09-13T14:19:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:07 smithi042 ceph-mon[29349]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:19:07.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:07 smithi042 ceph-mon[40707]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s, 0 objects/s recovering 2024-09-13T14:19:08.647 DEBUG:teuthology.orchestra.run.smithi042:node-exporter.a> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T14:19:08.650 DEBUG:teuthology.orchestra.run.smithi159:node-exporter.b> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T14:19:08.653 INFO:tasks.cephadm:Adding alertmanager.a on smithi042 2024-09-13T14:19:08.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi042=a' 2024-09-13T14:19:08.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:08 smithi159 ceph-mon[33330]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi042=a;smithi159=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:08.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:08 smithi159 ceph-mon[33330]: Saving service node-exporter spec with placement smithi042=a;smithi159=b;count:2 2024-09-13T14:19:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:08 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:08 smithi159 ceph-mon[33330]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:19:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[29349]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi042=a;smithi159=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[29349]: Saving service node-exporter spec with placement smithi042=a;smithi159=b;count:2 2024-09-13T14:19:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[29349]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:19:08.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[40707]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi042=a;smithi159=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[40707]: Saving service node-exporter spec with placement smithi042=a;smithi159=b;count:2 2024-09-13T14:19:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:08 smithi042 ceph-mon[40707]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T14:19:08.885 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:10 smithi159 ceph-mon[33330]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T14:19:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:10 smithi042 ceph-mon[29349]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T14:19:10.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:10 smithi042 ceph-mon[40707]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T14:19:12.043 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled alertmanager update... 2024-09-13T14:19:13.263 DEBUG:teuthology.orchestra.run.smithi042:alertmanager.a> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T14:19:13.266 INFO:tasks.cephadm:Adding grafana.a on smithi159 2024-09-13T14:19:13.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi159=a' 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[29349]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[29349]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[29349]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[40707]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[40707]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-09-13T14:19:13.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:13.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:13 smithi042 ceph-mon[40707]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:19:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:13 smithi159 ceph-mon[33330]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi042=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:13 smithi159 ceph-mon[33330]: Saving service alertmanager spec with placement smithi042=a;count:1 2024-09-13T14:19:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:13 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:13 smithi159 ceph-mon[33330]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T14:19:13.667 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:14.422 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:14 smithi159 ceph-mon[33330]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:14.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:14 smithi042 ceph-mon[29349]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:14 smithi042 ceph-mon[40707]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:16.369 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:16 smithi159 ceph-mon[33330]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:16 smithi042 ceph-mon[29349]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:16 smithi042 ceph-mon[40707]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:17.726 INFO:teuthology.orchestra.run.smithi159.stdout:Scheduled grafana update... 2024-09-13T14:19:18.421 DEBUG:teuthology.orchestra.run.smithi159:grafana.a> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T14:19:18.424 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T14:19:18.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:19:18.640 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:18.658 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:19:18.613Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:19:18.659 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:19:18 smithi159 bash[62549]: a615bf639f4bc6b4a73a94bf905305815fdadc9435f96c1ae84ac2e4df4764c6 2024-09-13T14:19:18.998 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:19:18 smithi159 systemd[1]: Started Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: Saving service grafana spec with placement smithi159=a;count:1 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:18 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: Saving service grafana spec with placement smithi159=a;count:1 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.014 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi159=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:19:19.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: Saving service grafana spec with placement smithi159=a;count:1 2024-09-13T14:19:19.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:19.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:18 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.759 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.759 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:19:19.759 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.759 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:19:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' 2024-09-13T14:19:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T14:19:19.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: ignoring --setuser ceph since I am not root 2024-09-13T14:19:19.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: ignoring --setgroup ceph since I am not root 2024-09-13T14:19:19.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-mgr[35405]: -- 172.21.15.159:0/738735375 <== mon.1 v2:172.21.15.159:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5614c579cd00 con 0x5614c5771400 2024-09-13T14:19:19.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:19.869+0000 7fab0470a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:19:19.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:19.924+0000 7fab0470a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:19:20.033 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setuser ceph since I am not root 2024-09-13T14:19:20.033 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setgroup ceph since I am not root 2024-09-13T14:19:20.033 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:19.898+0000 7f7d85521040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:19:20.033 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:19.968+0000 7f7d85521040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:19:20.314 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.123+0000 7f7d85521040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:19:20.314 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.198+0000 7f7d85521040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:19:20.416 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.070+0000 7fab0470a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:19:20.416 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.134+0000 7fab0470a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:19:20.417 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.277+0000 7fab0470a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:19:20.417 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.415+0000 7fab0470a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:19:20.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.350+0000 7f7d85521040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:19:20.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.499+0000 7f7d85521040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:19:20.656 INFO:teuthology.orchestra.run.smithi042.stdout:[client.0] 2024-09-13T14:19:20.656 INFO:teuthology.orchestra.run.smithi042.stdout: key = AQDoSeRmrIrmJhAA8JgiImv6sKnYrIX9oibtBA== 2024-09-13T14:19:20.975 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[40707]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:19:20.975 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[40707]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:19:20.975 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:20.975 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[29349]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[29349]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.831+0000 7f7d85521040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:19:20.976 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:20.895+0000 7f7d85521040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:19:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-mon[33330]: from='mgr.14150 172.21.15.42:0/3978844617' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T14:19:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-mon[33330]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-13T14:19:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3898338582' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.717+0000 7fab0470a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:19:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.777+0000 7fab0470a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:19:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:20.956+0000 7fab0470a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:19:21.342 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:21 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:21.086+0000 7f7d85521040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:19:21.364 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T14:19:21.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T14:19:21.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T14:19:21.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T14:19:21.605 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:19:21.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:21.844+0000 7fab0470a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:19:21.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:21.913+0000 7fab0470a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:19:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:21 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:21.997+0000 7f7d85521040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:19:22.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:22.065+0000 7f7d85521040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:19:22.740 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:22.622+0000 7fab0470a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:19:22.740 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:22.739+0000 7fab0470a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:19:22.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:22.892+0000 7fab0470a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:19:22.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:22.954+0000 7fab0470a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:19:23.083 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:22.812+0000 7f7d85521040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:19:23.084 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:22.934+0000 7f7d85521040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:19:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.081+0000 7f7d85521040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:19:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.145+0000 7f7d85521040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:19:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.297+0000 7f7d85521040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:19:23.401 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.107+0000 7fab0470a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:19:23.401 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.177+0000 7fab0470a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:19:23.401 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.240+0000 7fab0470a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:19:23.402 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.400+0000 7fab0470a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:19:23.471 INFO:teuthology.orchestra.run.smithi159.stdout:[client.1] 2024-09-13T14:19:23.471 INFO:teuthology.orchestra.run.smithi159.stdout: key = AQDrSeRmcj7bGxAAeOLjPynq5977tjrQiIPV7Q== 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[40707]: from='client.? 172.21.15.159:0/2888412353' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[29349]: from='client.? 172.21.15.159:0/2888412353' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:23.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.368+0000 7f7d85521040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:19:23.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.430+0000 7f7d85521040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:19:23.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.594+0000 7f7d85521040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:19:23.665 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.661+0000 7f7d85521040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:19:23.679 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-mon[33330]: from='client.? 172.21.15.159:0/2888412353' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.679 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T14:19:23.679 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T14:19:23.679 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.467+0000 7fab0470a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:19:23.679 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.538+0000 7fab0470a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:19:23.680 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.620+0000 7fab0470a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:19:23.941 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.730+0000 7f7d85521040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:19:23.941 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.816+0000 7f7d85521040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:19:23.941 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.877+0000 7f7d85521040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:19:23.941 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:23.938+0000 7f7d85521040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:19:23.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.678+0000 7fab0470a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:19:23.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:23.743+0000 7fab0470a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:19:24.111 DEBUG:teuthology.orchestra.run.smithi159:> set -ex 2024-09-13T14:19:24.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T14:19:24.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T14:19:24.150 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T14:19:24.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T14:19:24.358 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:24.780 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: Standby manager daemon x restarted 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: Standby manager daemon x restarted 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: Standby manager daemon x started 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:19:24.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: Standby manager daemon x restarted 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:19:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/2718106742' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:24.528+0000 7fab0470a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: [13/Sep/2024:14:19:24] ENGINE Bus STARTING 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: CherryPy Checker: 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: The Application mounted at '' has an empty config. 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: [13/Sep/2024:14:19:24] ENGINE Serving on http://:::9283 2024-09-13T14:19:25.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:19:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[35382]: [13/Sep/2024:14:19:24] ENGINE Bus STARTED 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:24.778+0000 7f7d85521040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:24] ENGINE Bus STARTING 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: CherryPy Checker: 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: The Application mounted at '' has an empty config. 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:25.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:24] ENGINE Serving on http://:::9283 2024-09-13T14:19:25.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:24 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:24] ENGINE Bus STARTED 2024-09-13T14:19:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:19:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: Active manager daemon y restarted 2024-09-13T14:19:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: Activating manager daemon y 2024-09-13T14:19:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: mgrmap e18: y(active, starting, since 0.00710076s), standbys: x 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:19:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: Manager daemon y is now available 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:25.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: Active manager daemon y restarted 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: Activating manager daemon y 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: mgrmap e18: y(active, starting, since 0.00710076s), standbys: x 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:25.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: Manager daemon y is now available 2024-09-13T14:19:25.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:25.853 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:25.938 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: Active manager daemon y restarted 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: Activating manager daemon y 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: mgrmap e18: y(active, starting, since 0.00710076s), standbys: x 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:19:25.939 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: Manager daemon y is now available 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:19:25.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:25.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:25.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:19:25.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:25.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:19:26.906 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'rbd' created 2024-09-13T14:19:27.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:26 smithi159 ceph-mon[33330]: mgrmap e19: y(active, since 1.01105s), standbys: x 2024-09-13T14:19:27.319 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:26 smithi042 ceph-mon[40707]: mgrmap e19: y(active, since 1.01105s), standbys: x 2024-09-13T14:19:27.320 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:26 smithi042 ceph-mon[29349]: mgrmap e19: y(active, since 1.01105s), standbys: x 2024-09-13T14:19:27.640 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T14:19:27.875 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3790516207' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:19:28.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 4016M 2024-09-13T14:19:28.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3790516207' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 4016M 2024-09-13T14:19:28.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:27 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3790516207' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 4016M 2024-09-13T14:19:28.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:27 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:28.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:28.315+0000 7f7cd9a48640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:19:28] ENGINE Error in HTTPServer.serve 2024-09-13T14:19:28.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: Traceback (most recent call last): 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._connections.run(self.expiration_interval) 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._run(expiration_interval) 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: s = self.context.wrap_socket( 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: return self.sslsocket_class._create( 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:19:28.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self.do_handshake() 2024-09-13T14:19:28.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:19:28.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: self._sslobj.do_handshake() 2024-09-13T14:19:28.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:19:28.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:19:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:19:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: [13/Sep/2024:14:19:28] ENGINE Bus STARTING 2024-09-13T14:19:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: [13/Sep/2024:14:19:28] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:19:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: [13/Sep/2024:14:19:28] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: [13/Sep/2024:14:19:28] ENGINE Bus STARTED 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[40707]: [13/Sep/2024:14:19:28] ENGINE Error in HTTPServer.serve 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:19:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: self.do_handshake() 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:19:29.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: [13/Sep/2024:14:19:28] ENGINE Bus STARTING 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: [13/Sep/2024:14:19:28] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: [13/Sep/2024:14:19:28] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: [13/Sep/2024:14:19:28] ENGINE Bus STARTED 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:28 smithi042 ceph-mon[29349]: [13/Sep/2024:14:19:28] ENGINE Error in HTTPServer.serve 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:19:29.101 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:19:29.102 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:19:29.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: [13/Sep/2024:14:19:28] ENGINE Bus STARTING 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: [13/Sep/2024:14:19:28] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: [13/Sep/2024:14:19:28] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: [13/Sep/2024:14:19:28] ENGINE Bus STARTED 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:28 smithi159 ceph-mon[33330]: [13/Sep/2024:14:19:28] ENGINE Error in HTTPServer.serve 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout: Traceback (most recent call last): 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:19:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._run(expiration_interval) 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: s = self.context.wrap_socket( 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: return self.sslsocket_class._create( 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: self.do_handshake() 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:19:29.250 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._sslobj.do_handshake() 2024-09-13T14:19:29.251 INFO:journalctl@ceph.mon.b.smithi159.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/665736267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/665736267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:29 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-13T14:19:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/665736267' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:29 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:19:31.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:19:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:30 smithi159 ceph-mon[33330]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:19:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[29349]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T14:19:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:30 smithi042 ceph-mon[40707]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T14:19:32.228 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[40707]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:32.228 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[40707]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[40707]: Cluster is now healthy 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[40707]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[29349]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[29349]: Cluster is now healthy 2024-09-13T14:19:32.229 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:31 smithi042 ceph-mon[29349]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:19:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:31 smithi159 ceph-mon[33330]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:31 smithi159 ceph-mon[33330]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:19:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:31 smithi159 ceph-mon[33330]: Cluster is now healthy 2024-09-13T14:19:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:31 smithi159 ceph-mon[33330]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T14:19:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:33 smithi042 ceph-mon[29349]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:19:33.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:33 smithi042 ceph-mon[40707]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:19:33.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:19:33] "GET /metrics HTTP/1.1" 200 23680 "" "Prometheus/2.43.0" 2024-09-13T14:19:33.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:33 smithi159 ceph-mon[33330]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T14:19:33.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T14:19:33.999 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:34.389 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:34 smithi042 ceph-mon[40707]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-13T14:19:34.389 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:34 smithi042 ceph-mon[29349]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-13T14:19:34.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:34 smithi159 ceph-mon[33330]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-09-13T14:19:36.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:36 smithi159 ceph-mon[33330]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T14:19:36.508 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:36 smithi042 ceph-mon[40707]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T14:19:36.508 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:36 smithi042 ceph-mon[29349]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-13T14:19:38.097 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:19:38.750 INFO:teuthology.orchestra.run.smithi042.stdout:[{"version":20,"timestamp":"2024-09-13T14:19:27.705008+0000","name":"","changes":[{"name":"osd/host:smithi159/osd_memory_target","previous_value":"4479547596","new_value":"4211112140"}]}] 2024-09-13T14:19:38.750 INFO:tasks.ceph_manager:config epoch is 20 2024-09-13T14:19:38.750 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:19:38.751 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:19:38.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:19:38.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:38 smithi159 ceph-mon[33330]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 311 B/s wr, 17 op/s 2024-09-13T14:19:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:38 smithi159 ceph-mon[33330]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:19:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:38 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2966058759' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[40707]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 311 B/s wr, 17 op/s 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[40707]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2966058759' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[29349]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 311 B/s wr, 17 op/s 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[29349]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T14:19:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:38 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2966058759' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T14:19:39.285 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:40.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:40 smithi042 ceph-mon[29349]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-09-13T14:19:40.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:40 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:40.984 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:40 smithi042 ceph-mon[40707]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-09-13T14:19:40.984 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:40 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:40 smithi159 ceph-mon[33330]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 270 B/s wr, 0 op/s 2024-09-13T14:19:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:40 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:41.501 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:19:41.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:41 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/138084509' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:19:42.097 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:41 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/138084509' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:19:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:41 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/138084509' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 238 B/s wr, 0 op/s 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:42.692 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE Bus STOPPING 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE Bus STOPPED 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE Bus STARTING 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 238 B/s wr, 0 op/s 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.693 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.694 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:42.694 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 238 B/s wr, 0 op/s 2024-09-13T14:19:42.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:42 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:42.899 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":21,"flags":0,"active_gid":14442,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6800","nonce":3689509320},{"type":"v1","addr":"172.21.15.42:6801","nonce":3689509320}]},"active_addr":"172.21.15.42:6801/3689509320","active_change":"2024-09-13T14:19:24.781610+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":471235645}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":800406081}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":1962929896}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":3559778733}]}]} 2024-09-13T14:19:42.903 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:19:42.903 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:19:42.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:19:43.078 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE Serving on http://:::9283 2024-09-13T14:19:43.078 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:19:42] ENGINE Bus STARTED 2024-09-13T14:19:43.078 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:19:43] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-09-13T14:19:43.108 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[40707]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[40707]: Deploying daemon grafana.a on smithi159 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[29349]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:43.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:43 smithi042 ceph-mon[29349]: Deploying daemon grafana.a on smithi159 2024-09-13T14:19:43.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:43 smithi159 ceph-mon[33330]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T14:19:43.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:43 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T14:19:43.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:43 smithi159 ceph-mon[33330]: Deploying daemon grafana.a on smithi159 2024-09-13T14:19:44.318 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:19:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[71506]: ts=2024-09-13T14:19:44.168Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000583843s 2024-09-13T14:19:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:44 smithi042 ceph-mon[29349]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:44.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:44 smithi042 ceph-mon[40707]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:44.966 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:19:44.966 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":66,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:19:37.648336+0000","last_up_change":"2024-09-13T14:18:55.210301+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6803","nonce":917582203}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6805","nonce":917582203}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6809","nonce":917582203}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6807","nonce":917582203}]},"public_addr":"172.21.15.42:6803/917582203","cluster_addr":"172.21.15.42:6805/917582203","heartbeat_back_addr":"172.21.15.42:6809/917582203","heartbeat_front_addr":"172.21.15.42:6807/917582203","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6811","nonce":3832896807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6813","nonce":3832896807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6817","nonce":3832896807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6815","nonce":3832896807}]},"public_addr":"172.21.15.42:6811/3832896807","cluster_addr":"172.21.15.42:6813/3832896807","heartbeat_back_addr":"172.21.15.42:6817/3832896807","heartbeat_front_addr":"172.21.15.42:6815/3832896807","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6819","nonce":2844866888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6821","nonce":2844866888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6825","nonce":2844866888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6823","nonce":2844866888}]},"public_addr":"172.21.15.42:6819/2844866888","cluster_addr":"172.21.15.42:6821/2844866888","heartbeat_back_addr":"172.21.15.42:6825/2844866888","heartbeat_front_addr":"172.21.15.42:6823/2844866888","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6827","nonce":38876887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6829","nonce":38876887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6833","nonce":38876887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6831","nonce":38876887}]},"public_addr":"172.21.15.42:6827/38876887","cluster_addr":"172.21.15.42:6829/38876887","heartbeat_back_addr":"172.21.15.42:6833/38876887","heartbeat_front_addr":"172.21.15.42:6831/38876887","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:19:44.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:44 smithi159 ceph-mon[33330]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:45.648 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:19:45.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:19:45.856 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:45.975 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[29349]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:45.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:45.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/929943051' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:45.976 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[40707]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:45.976 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:45.976 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:45 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/929943051' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:45 smithi159 ceph-mon[33330]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T14:19:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:45 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:45 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/929943051' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:47.945 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:19:47.945 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":66,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:19:37.648336+0000","last_up_change":"2024-09-13T14:18:55.210301+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6803","nonce":917582203}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6805","nonce":917582203}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6809","nonce":917582203}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6807","nonce":917582203}]},"public_addr":"172.21.15.42:6803/917582203","cluster_addr":"172.21.15.42:6805/917582203","heartbeat_back_addr":"172.21.15.42:6809/917582203","heartbeat_front_addr":"172.21.15.42:6807/917582203","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6811","nonce":3832896807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6813","nonce":3832896807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6817","nonce":3832896807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6815","nonce":3832896807}]},"public_addr":"172.21.15.42:6811/3832896807","cluster_addr":"172.21.15.42:6813/3832896807","heartbeat_back_addr":"172.21.15.42:6817/3832896807","heartbeat_front_addr":"172.21.15.42:6815/3832896807","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6819","nonce":2844866888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6821","nonce":2844866888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6825","nonce":2844866888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6823","nonce":2844866888}]},"public_addr":"172.21.15.42:6819/2844866888","cluster_addr":"172.21.15.42:6821/2844866888","heartbeat_back_addr":"172.21.15.42:6825/2844866888","heartbeat_front_addr":"172.21.15.42:6823/2844866888","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6827","nonce":38876887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6829","nonce":38876887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6833","nonce":38876887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6831","nonce":38876887}]},"public_addr":"172.21.15.42:6827/38876887","cluster_addr":"172.21.15.42:6829/38876887","heartbeat_back_addr":"172.21.15.42:6833/38876887","heartbeat_front_addr":"172.21.15.42:6831/38876887","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:19:47.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:47 smithi042 ceph-mon[29349]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:47.946 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:47 smithi042 ceph-mon[40707]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:47 smithi159 ceph-mon[33330]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:48.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:19:48.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:19:48.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:19:48.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:19:48.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:19:48.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:19:48.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:19:48.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:19:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:48 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1687444035' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:49.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:48 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1687444035' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:49.103 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.103 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.106 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.107 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.108 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.108 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.109 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.110 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:19:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:48 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1687444035' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:19:50.172 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:49 smithi042 ceph-mon[29349]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:50.173 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:49 smithi042 ceph-mon[40707]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:49 smithi159 ceph-mon[33330]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:52.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:51 smithi042 ceph-mon[29349]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:52.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:51 smithi042 ceph-mon[40707]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:52.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:51 smithi159 ceph-mon[33330]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:52.348 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:19:52 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[71506]: ts=2024-09-13T14:19:52.171Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002852207s 2024-09-13T14:19:53.126 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:19:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:19:53] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-09-13T14:19:53.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:53 smithi159 systemd[1]: Starting Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:19:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:53 smithi042 ceph-mon[29349]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:53 smithi042 ceph-mon[40707]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:53 smithi159 ceph-mon[33330]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:54.249 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:53 smithi159 podman[64688]: 2024-09-13 14:19:53.722190777 +0000 UTC m=+0.014886694 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-13T14:19:54.249 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:53 smithi159 podman[64688]: 2024-09-13 14:19:53.831858214 +0000 UTC m=+0.124554121 container create 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, version=8.5, release=239.1651231664, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-13T14:19:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:54.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 podman[64688]: 2024-09-13 14:19:54.729105382 +0000 UTC m=+1.021801299 container init 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, architecture=x86_64, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, release=239.1651231664, vcs-type=git, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, description=Ceph Grafana Container, name=ubi8, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-09-13T14:19:54.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 podman[64688]: 2024-09-13 14:19:54.734851627 +0000 UTC m=+1.027547534 container start 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896451036Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896653376Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896673753Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896681593Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896687732Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896694488Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896700283Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896706315Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896712797Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896718784Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896724458Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:19:55.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896730074Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896735863Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=settings t=2024-09-13T14:19:54.896741722Z level=info msg="App mode production" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore t=2024-09-13T14:19:54.896791391Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore t=2024-09-13T14:19:54.896808526Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.897707548Z level=info msg="Starting DB migrations" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.897781616Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.898312844Z level=info msg="Executing migration" id="create user table" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.898759037Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.899612806Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.900612479Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.90137361Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.90221899Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T14:19:55.001 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.903072774Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.904068807Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.904639736Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.90779134Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.910667354Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.913663711Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.916907267Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.919843822Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.922742813Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.925582446Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.928131531Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.931649247Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.934882675Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.937474136Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T14:19:55.002 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.940466526Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.943157168Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.946198088Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.948922992Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.95176598Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.954556809Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.957416777Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.960281348Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.962929378Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T14:19:55.003 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:54.963329833Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T14:19:55.052 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:54 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:55.052 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:54 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.004435209Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.005081951Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.007789869Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.008201057Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.00858989Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.009014452Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.011429135Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.011810714Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.014305481Z level=info msg="Executing migration" id="create star table" 2024-09-13T14:19:55.258 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.017061128Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.020051526Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.02266626Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.025353593Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.028228207Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.028987367Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.029865943Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.030676994Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.031564251Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.032409773Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.033386096Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.034149745Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.035151904Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T14:19:55.259 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.035866601Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.036685064Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.037628738Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.038412467Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.03936924Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.040235827Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.041086385Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.041921132Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.0427109Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.043550799Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.044545341Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.045422952Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T14:19:55.260 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.046113143Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T14:19:55.261 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.046580312Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T14:19:55.261 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.0470263Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T14:19:55.261 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.047478079Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T14:19:55.262 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.047926611Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T14:19:55.262 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.0483245Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T14:19:55.262 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.048587766Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.048883365Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.049372226Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.050388454Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.051093549Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.051965735Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.052297304Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.052830869Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.053695082Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.056618772Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T14:19:55.263 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.059551682Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.062297637Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.06495876Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.067911727Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.070636611Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.073426891Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.076174343Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.078963657Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T14:19:55.264 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.081802101Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.084583341Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.087465911Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.090222182Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.092923457Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.095632911Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.098406127Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.10115619Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.103894496Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.106717323Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.109391102Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.11211261Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T14:19:55.265 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.114750027Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.117533863Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.12019134Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.1228163Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.125573358Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.128179588Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.130812615Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.133315518Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.135893149Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.138476568Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.141308168Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.14392006Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T14:19:55.266 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.146525654Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.147007648Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.149931857Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.152537704Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.155252164Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.157907488Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.160652504Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.163391901Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.164031243Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.164817321Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.165952914Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.166353415Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T14:19:55.267 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.166786029Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.167216179Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.168076254Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.168878186Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.169281636Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.169727528Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.170388701Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.170995174Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.171790078Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.172606764Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.173370219Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.173809332Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.174194666Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.175031422Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T14:19:55.268 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.175649802Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.17612626Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.176957075Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.177269584Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.178183229Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.178864385Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.179453735Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.180279756Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.1813409Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.181982426Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.182310763Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.182744037Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T14:19:55.269 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.183183224Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.183778539Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.184096372Z level=info msg="Executing migration" id="create session table" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.184546034Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.184868154Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.185165965Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.185604624Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.186023154Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.186287859Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.186556846Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.186826466Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.187095485Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T14:19:55.270 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.187477548Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.187776996Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.188345122Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.188643599Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.189201257Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.189790692Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.190087214Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.19049511Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.19095538Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.191394539Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.192253324Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.193168757Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.193832427Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T14:19:55.271 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.194200305Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.194607813Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.195008098Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.196390082Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.196775104Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.197193381Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.197483304Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.19785599Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.198232669Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.198876395Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.199481008Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.200156158Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.200779802Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T14:19:55.272 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.201263045Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.201541853Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.201842576Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.202242462Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.202668449Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.203042068Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.203400662Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.203813895Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.204429842Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.205108629Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.205423416Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.205851117Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.206246719Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T14:19:55.273 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.206883714Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.207182098Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.207575126Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.207974848Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.208258553Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.208673497Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.209473671Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.209933865Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.210339769Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.21083881Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.211691559Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.212015092Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.212715026Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.213559322Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.214402485Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T14:19:55.274 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.215266988Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.215753921Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.218490823Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.221302284Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.224175611Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.224794081Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.227777789Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.230601714Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.233261026Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.233982135Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.236664062Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.239457128Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.242320715Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T14:19:55.275 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.2426853Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.245571775Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.24825802Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.250997595Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.251595203Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.254501596Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.257061917Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.259742388Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T14:19:55.276 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.26019913Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.26295475Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.263405876Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.265691272Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.266129804Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.268415876Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T14:19:55.509 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.271035448Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.273746232Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.276430657Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.279210476Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.281896024Z level=info msg="Executing migration" id="create team table" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.284595719Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.28724229Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.287786683Z level=info msg="Executing migration" id="create team member table" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.290698005Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.293312621Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.294121481Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.294593274Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.295456726Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T14:19:55.510 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.296237444Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T14:19:55.511 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.297093564Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T14:19:55.511 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.297537486Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.298398397Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.299109873Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.299587661Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.300078897Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.300947926Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.301715874Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.302684619Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.303404218Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.304445458Z level=info msg="Executing migration" id="create tag table" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.305175202Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.306188311Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.306932811Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T14:19:55.512 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.307733517Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.308609944Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.310565369Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.311351783Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.312385194Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.313082417Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.313980922Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.314566715Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.314995977Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.315803059Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.316544695Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.317341651Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T14:19:55.513 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.318194641Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.319044026Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.319988779Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.320813271Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.32170816Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.322560899Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.325451858Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.328262274Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.330917579Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.333935355Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.336663877Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.339503855Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.342372309Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T14:19:55.514 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.344824134Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.347702611Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.350265225Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.352972111Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.355660662Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.358403811Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.361344682Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.36387899Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.366570509Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.369283957Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.372173947Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.374743142Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T14:19:55.515 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.377581364Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.380001908Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.382852138Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.385435004Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.388143137Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.390892842Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.393585354Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.396229672Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.39909887Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.401675205Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.413066325Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.414219669Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T14:19:55.516 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.416846745Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.419758884Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.423367768Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.427035924Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.42783569Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.428817693Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.429867481Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.430883946Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.43177895Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.432437229Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.433357599Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.434068619Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.434900214Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.43574832Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T14:19:55.517 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.436562766Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.43740585Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.438448629Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.439485524Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.440414564Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.441260293Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.442170263Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.443055391Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.444092232Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.444781011Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.445580954Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.446507789Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.447251252Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T14:19:55.518 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.448142968Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.449004246Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.449854102Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.450726942Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.451578749Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.452424333Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.453393804Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.454330177Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.455248253Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.45615534Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.457112488Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.45817576Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.461030605Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.463984043Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T14:19:55.519 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.466620994Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.469316096Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.472069784Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.474851345Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.477569496Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.480484148Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.483144626Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.486021715Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.486114238Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.488816046Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T14:19:55.520 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.491704161Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.494440778Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.494588688Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.494717414Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.497291738Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.50004727Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.502861821Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.505753261Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.50868511Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T14:19:55.521 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.511196923Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.514025763Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.516762504Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.519615883Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.522202224Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.526606127Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.52936676Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.532080155Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.5367434Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.540931449Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.543620219Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T14:19:55.782 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.546240679Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.548876171Z level=info msg="Executing migration" id="create permission table" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.551662144Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.552126993Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.552989624Z level=info msg="Executing migration" id="create role table" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.553845914Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.554782268Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.555719276Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T14:19:55.783 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.556572431Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T14:19:55.784 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.557434921Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T14:19:55.784 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.558506481Z level=info msg="Executing migration" id="create team role table" 2024-09-13T14:19:55.784 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.559230988Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T14:19:55.784 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.560221534Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T14:19:55.784 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.560937101Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.561792877Z level=info msg="Executing migration" id="create user role table" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.562657369Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.563500343Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.566269965Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.569089843Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.571937459Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.574562339Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.577524776Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.57850881Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.580218701Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.582986857Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.583577407Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T14:19:55.785 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.584016284Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.584404602Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.584859318Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.58586314Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.586305867Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.586763074Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.587069235Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 bash[64688]: 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.587496437Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.588031027Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.588375516Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.588700417Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.589053966Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.589622262Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T14:19:55.786 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.590064498Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 systemd[1]: Started Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.591085564Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.591393669Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.591827676Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.592263647Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.592703985Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.593686533Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.594149813Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.596875393Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.597000459Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.597320894Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.597428865Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:19:55.787 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.597749836Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.600437412Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.603237Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.606073047Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.609771412Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.611598289Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.6143664Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.617158013Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.619972396Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.622787325Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.625467876Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.628270014Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.631416875Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T14:19:55.788 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.634202508Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.636887513Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.63950645Z level=info msg="Executing migration" id="add share column" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.642269496Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.644951902Z level=info msg="Executing migration" id="create file table" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.650387632Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.65084909Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.653584341Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.653993603Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.65947384Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.659826306Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.662595397Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.662955652Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.665273223Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T14:19:55.789 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.667942947Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.668743628Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.669580728Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.670408499Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.671387856Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.672216391Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.673431596Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.674544517Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.675324619Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.685101311Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.685939661Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.686774351Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.690171361Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T14:19:55.790 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.690937529Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.691787223Z level=info msg="Executing migration" id="create folder table" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.692601116Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.693460012Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.694409945Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.695106921Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=migrator t=2024-09-13T14:19:55.695978535Z level=info msg="migrations completed" performed=485 skipped=0 duration=798.216414ms 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore t=2024-09-13T14:19:55.696393773Z level=info msg="Created default organization" 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=plugin.loader t=2024-09-13T14:19:55.753053645Z level=info msg="Plugin registered" pluginID=input 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=plugin.loader t=2024-09-13T14:19:55.781488448Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=plugin.loader t=2024-09-13T14:19:55.781524605Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=secrets t=2024-09-13T14:19:55.781559113Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:19:55.791 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=query_data t=2024-09-13T14:19:55.783373065Z level=info msg="Query Service initialization" 2024-09-13T14:19:56.248 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=live.push_http t=2024-09-13T14:19:55.785848327Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:19:56.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-mon[33330]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:19:56.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=infra.usagestats.collector t=2024-09-13T14:19:56.518148807Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:19:56.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=provisioning.datasources t=2024-09-13T14:19:56.572582236Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:19:56.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=provisioning.datasources t=2024-09-13T14:19:56.604296755Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T14:19:56.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=provisioning.alerting t=2024-09-13T14:19:56.625824162Z level=info msg="starting to provision alerting" 2024-09-13T14:19:56.999 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=provisioning.alerting t=2024-09-13T14:19:56.62590146Z level=info msg="finished to provision alerting" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=grafanaStorageLogger t=2024-09-13T14:19:56.629635242Z level=info msg="storage starting" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=ngalert.state.manager t=2024-09-13T14:19:56.629643656Z level=info msg="Warming state cache for startup" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=http.server t=2024-09-13T14:19:56.629754142Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=ngalert.state.manager t=2024-09-13T14:19:56.640027882Z level=info msg="State cache has been initialized" states=0 duration=10.369949ms 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:19:56.640078597Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=ticker t=2024-09-13T14:19:56.640104757Z level=info msg=starting first_tick=2024-09-13T14:20:00Z 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore.transactions t=2024-09-13T14:19:56.640151902Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore.transactions t=2024-09-13T14:19:56.681401874Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore.transactions t=2024-09-13T14:19:56.723750258Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:19:57.000 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:19:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[64702]: logger=sqlstore.transactions t=2024-09-13T14:19:56.734960965Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[29349]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:19:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:56 smithi042 ceph-mon[40707]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:19:58.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:19:58 smithi159 ceph-mon[33330]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:19:58 smithi042 ceph-mon[29349]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:19:58 smithi042 ceph-mon[40707]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:19:59.793 INFO:teuthology.orchestra.run.smithi042.stdout:55834574890 2024-09-13T14:19:59.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:20:00.346 INFO:teuthology.orchestra.run.smithi042.stdout:34359738415 2024-09-13T14:20:00.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:20:00.933 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[29349]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:00.933 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:00.933 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[29349]: overall HEALTH_OK 2024-09-13T14:20:00.934 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[40707]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:00.934 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:00.934 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:00 smithi042 ceph-mon[40707]: overall HEALTH_OK 2024-09-13T14:20:00.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:00 smithi159 ceph-mon[33330]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:00 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:00 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:20:01.642 INFO:teuthology.orchestra.run.smithi042.stdout:107374182433 2024-09-13T14:20:01.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:20:02.348 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:02 smithi042 systemd[1]: Starting Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:02 smithi042 ceph-mon[29349]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:02 smithi042 ceph-mon[40707]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:02.915 INFO:teuthology.orchestra.run.smithi042.stdout:77309411366 2024-09-13T14:20:02.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:20:02.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:02 smithi159 ceph-mon[33330]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:03.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:20:03] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-09-13T14:20:03.843 INFO:teuthology.orchestra.run.smithi042.stdout:163208757273 2024-09-13T14:20:03.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:20:04.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:04 smithi159 ceph-mon[33330]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:05.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:04 smithi042 ceph-mon[29349]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:04 smithi042 ceph-mon[40707]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:05.883 INFO:teuthology.orchestra.run.smithi042.stdout:128849018910 2024-09-13T14:20:05.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:20:06.421 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:06.424 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:06.930 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:06 smithi042 ceph-mon[29349]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:06.930 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:06 smithi042 ceph-mon[40707]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:06.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:06 smithi159 ceph-mon[33330]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:08.836 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:08 smithi042 ceph-mon[29349]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:08.837 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:08 smithi042 ceph-mon[40707]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:08.837 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:08 smithi042 bash[74247]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:20:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:08 smithi159 ceph-mon[33330]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:09.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:08 smithi042 bash[74247]: Getting image source signatures 2024-09-13T14:20:09.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:08 smithi042 bash[74247]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:20:09.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:08 smithi042 bash[74247]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:20:09.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:08 smithi042 bash[74247]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:20:09.293 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:09.317 INFO:teuthology.orchestra.run.smithi042.stdout:197568495636 2024-09-13T14:20:09.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:20:09.883 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:09.885 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:09.885 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:10.768 INFO:teuthology.orchestra.run.smithi042.stdout:231928233999 2024-09-13T14:20:10.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:20:10.792 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:10 smithi042 ceph-mon[29349]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:10.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:10 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:10.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:10 smithi042 ceph-mon[40707]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:10.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:10 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:10.793 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:10 smithi042 bash[74247]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:20:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:10 smithi159 ceph-mon[33330]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:10 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:11.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:10 smithi042 bash[74247]: Writing manifest to image destination 2024-09-13T14:20:11.766 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:11 smithi042 podman[74247]: 2024-09-13 14:20:11.424269603 +0000 UTC m=+3.028273045 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:20:12.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:11 smithi042 podman[74247]: 2024-09-13 14:20:11.866300471 +0000 UTC m=+3.470303902 container create bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:12.405 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 podman[74247]: 2024-09-13 14:20:12.212088187 +0000 UTC m=+3.816091628 container init bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 podman[74247]: 2024-09-13 14:20:12.217612543 +0000 UTC m=+3.821615982 container start bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.221Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.221Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.222Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.222Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.222Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.222Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:20:12.406 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:20:12.407 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:20:12.408 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:20:12.409 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:20:12.410 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[74573]: ts=2024-09-13T14:20:12.223Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 bash[74247]: bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e 2024-09-13T14:20:12.411 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:20:12 smithi042 systemd[1]: Started Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[40707]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[29349]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:12.930 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:12 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:12 smithi159 ceph-mon[33330]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:12.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:12 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:12 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:12.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:12 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:13.186 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 systemd[1]: Starting Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:13.209 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:20:13] "GET /metrics HTTP/1.1" 200 28508 "" "Prometheus/2.43.0" 2024-09-13T14:20:13.498 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 bash[64980]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-13T14:20:13.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:13 smithi042 ceph-mon[29349]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:20:13.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:13 smithi042 ceph-mon[40707]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:20:13.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:13 smithi159 ceph-mon[33330]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:20:13.998 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 bash[64980]: Getting image source signatures 2024-09-13T14:20:13.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 bash[64980]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-13T14:20:13.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 bash[64980]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-13T14:20:13.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:13 smithi159 bash[64980]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-13T14:20:14.939 INFO:teuthology.orchestra.run.smithi042.stdout:163208757276 2024-09-13T14:20:14.939 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:14 smithi042 ceph-mon[29349]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:14.939 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:14 smithi042 ceph-mon[40707]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:14.949 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:14 smithi159 ceph-mon[33330]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:15.248 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:14 smithi159 bash[64980]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-13T14:20:15.248 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 bash[64980]: Writing manifest to image destination 2024-09-13T14:20:15.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1390016600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:20:15.707 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 podman[64980]: 2024-09-13 14:20:15.364450419 +0000 UTC m=+2.142128931 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T14:20:15.707 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 podman[64980]: 2024-09-13 14:20:15.471331076 +0000 UTC m=+2.249009620 container create 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:20:15.707 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 podman[64980]: 2024-09-13 14:20:15.706781361 +0000 UTC m=+2.484459861 container init 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:20:15.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:15 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1390016600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:20:15.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:15 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1390016600' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 podman[64980]: 2024-09-13 14:20:15.711708476 +0000 UTC m=+2.489386992 container start 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.715Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.715Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.716Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.716Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.716Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.716Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:20:15.999 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:20:16.000 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:20:16.001 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:20:16.002 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:20:16.003 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[65056]: ts=2024-09-13T14:20:15.717Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 bash[64980]: 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e 2024-09-13T14:20:16.004 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:20:15 smithi159 systemd[1]: Started Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:16.550 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757273 got 163208757276 for osd.5 2024-09-13T14:20:16.550 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:16 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:16.837 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:16.837 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.837 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:16.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:16 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:16.899 INFO:teuthology.orchestra.run.smithi042.stdout:107374182436 2024-09-13T14:20:17.468 INFO:teuthology.orchestra.run.smithi042.stdout:55834574893 2024-09-13T14:20:17.779 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:17 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3092721040' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:20:17.779 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:17 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2683796780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:20:17.779 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:17 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3092721040' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:20:17.780 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:17 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2683796780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:20:17.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:17 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3092721040' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:20:17.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:17 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2683796780' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:20:18.206 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574893 for osd.1 2024-09-13T14:20:18.206 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:18.493 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182433 got 107374182436 for osd.3 2024-09-13T14:20:18.493 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:18.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:18 smithi042 ceph-mon[29349]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:18 smithi042 ceph-mon[40707]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:18 smithi159 ceph-mon[33330]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:19.355 INFO:teuthology.orchestra.run.smithi042.stdout:77309411370 2024-09-13T14:20:19.580 INFO:teuthology.orchestra.run.smithi042.stdout:128849018913 2024-09-13T14:20:19.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2875959347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:20:19.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1421031398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:20:19.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2875959347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:20:19.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:19 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1421031398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:20:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:19 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2875959347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:20:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:19 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:19 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:19 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1421031398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:20:20.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018910 got 128849018913 for osd.4 2024-09-13T14:20:20.187 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:20.331 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:20 smithi042 ceph-mon[29349]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:20.976 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:20 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:20.977 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:20 smithi042 ceph-mon[40707]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:20.977 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:20 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:20 smithi159 ceph-mon[33330]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:20 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:21.005 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411370 for osd.2 2024-09-13T14:20:21.005 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:22.017 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:22.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:22 smithi159 ceph-mon[33330]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:23.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:22 smithi042 ceph-mon[40707]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:23.059 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:22 smithi042 ceph-mon[29349]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:23.319 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:20:23] "GET /metrics HTTP/1.1" 200 28508 "" "Prometheus/2.43.0" 2024-09-13T14:20:24.797 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:24 smithi042 ceph-mon[29349]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:24.797 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:24 smithi042 ceph-mon[40707]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:24 smithi159 ceph-mon[33330]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:25.153 INFO:teuthology.orchestra.run.smithi042.stdout:34359738421 2024-09-13T14:20:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:25 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3018215061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:20:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:25 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3018215061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:20:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:25 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3018215061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:20:26.184 INFO:teuthology.orchestra.run.smithi042.stdout:197568495640 2024-09-13T14:20:26.317 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738421 for osd.0 2024-09-13T14:20:26.318 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:26.909 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:26 smithi042 ceph-mon[40707]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:26.909 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:26 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/649017486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:20:26.909 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:26 smithi042 ceph-mon[29349]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:26.910 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:26 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/649017486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:20:26.931 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495636 got 197568495640 for osd.6 2024-09-13T14:20:26.931 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:26.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:26 smithi159 ceph-mon[33330]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:26 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/649017486' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:20:27.579 INFO:teuthology.orchestra.run.smithi042.stdout:231928234002 2024-09-13T14:20:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:27 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2269221530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:20:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:27 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2269221530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:20:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:27 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2269221530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:20:28.953 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:28 smithi042 ceph-mon[40707]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:28.953 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:28 smithi042 ceph-mon[29349]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:28.979 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234002 for osd.7 2024-09-13T14:20:28.979 DEBUG:teuthology.parallel:result is None 2024-09-13T14:20:28.979 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:20:28.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:20:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:28 smithi159 ceph-mon[33330]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:29.724 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:30.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:30.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:30 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:30 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:31.746 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:31 smithi042 ceph-mon[40707]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:20:31.746 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:31 smithi042 ceph-mon[40707]: Reconfiguring daemon alertmanager.a on smithi042 2024-09-13T14:20:31.746 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:31 smithi042 ceph-mon[29349]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:20:31.747 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:31 smithi042 ceph-mon[29349]: Reconfiguring daemon alertmanager.a on smithi042 2024-09-13T14:20:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:31 smithi159 ceph-mon[33330]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T14:20:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:31 smithi159 ceph-mon[33330]: Reconfiguring daemon alertmanager.a on smithi042 2024-09-13T14:20:32.367 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:20:32.368 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:20:32.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:32 smithi159 ceph-mon[33330]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:33.019 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:32 smithi042 ceph-mon[40707]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:33.020 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:32 smithi042 ceph-mon[29349]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:33.047 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-13T14:20:30.803504+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227236,"kb_used_data":6988,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733980,"statfs":{"total":767960285184,"available":767727595520,"internally_reserved":0,"allocated":7155712,"data_stored":4035028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":225431097},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002139"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.995309+0000","last_change":"2024-09-13T14:19:32.224331+0000","last_active":"2024-09-13T14:19:37.995309+0000","last_peered":"2024-09-13T14:19:37.995309+0000","last_clean":"2024-09-13T14:19:37.995309+0000","last_became_active":"2024-09-13T14:19:27.914929+0000","last_became_peered":"2024-09-13T14:19:27.914929+0000","last_unstale":"2024-09-13T14:19:37.995309+0000","last_undegraded":"2024-09-13T14:19:37.995309+0000","last_fullsized":"2024-09-13T14:19:37.995309+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:47.216732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.739681+0000","last_change":"2024-09-13T14:19:32.224489+0000","last_active":"2024-09-13T14:19:37.739681+0000","last_peered":"2024-09-13T14:19:37.739681+0000","last_clean":"2024-09-13T14:19:37.739681+0000","last_became_active":"2024-09-13T14:19:28.303467+0000","last_became_peered":"2024-09-13T14:19:28.303467+0000","last_unstale":"2024-09-13T14:19:37.739681+0000","last_undegraded":"2024-09-13T14:19:37.739681+0000","last_fullsized":"2024-09-13T14:19:37.739681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:47.015688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032689699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.752693+0000","last_change":"2024-09-13T14:19:32.403220+0000","last_active":"2024-09-13T14:19:37.752693+0000","last_peered":"2024-09-13T14:19:37.752693+0000","last_clean":"2024-09-13T14:19:37.752693+0000","last_became_active":"2024-09-13T14:19:28.303245+0000","last_became_peered":"2024-09-13T14:19:28.303245+0000","last_unstale":"2024-09-13T14:19:37.752693+0000","last_undegraded":"2024-09-13T14:19:37.752693+0000","last_fullsized":"2024-09-13T14:19:37.752693+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:32.196419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.739665+0000","last_change":"2024-09-13T14:19:32.224472+0000","last_active":"2024-09-13T14:19:37.739665+0000","last_peered":"2024-09-13T14:19:37.739665+0000","last_clean":"2024-09-13T14:19:37.739665+0000","last_became_active":"2024-09-13T14:19:27.913130+0000","last_became_peered":"2024-09-13T14:19:27.913130+0000","last_unstale":"2024-09-13T14:19:37.739665+0000","last_undegraded":"2024-09-13T14:19:37.739665+0000","last_fullsized":"2024-09-13T14:19:37.739665+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:25.471342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.994998+0000","last_change":"2024-09-13T14:19:32.223637+0000","last_active":"2024-09-13T14:19:37.994998+0000","last_peered":"2024-09-13T14:19:37.994998+0000","last_clean":"2024-09-13T14:19:37.994998+0000","last_became_active":"2024-09-13T14:19:27.912799+0000","last_became_peered":"2024-09-13T14:19:27.912799+0000","last_unstale":"2024-09-13T14:19:37.994998+0000","last_undegraded":"2024-09-13T14:19:37.994998+0000","last_fullsized":"2024-09-13T14:19:37.994998+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:07.658550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.651457+0000","last_change":"2024-09-13T14:19:32.219943+0000","last_active":"2024-09-13T14:19:37.651457+0000","last_peered":"2024-09-13T14:19:37.651457+0000","last_clean":"2024-09-13T14:19:37.651457+0000","last_became_active":"2024-09-13T14:19:28.274350+0000","last_became_peered":"2024-09-13T14:19:28.274350+0000","last_unstale":"2024-09-13T14:19:37.651457+0000","last_undegraded":"2024-09-13T14:19:37.651457+0000","last_fullsized":"2024-09-13T14:19:37.651457+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:22.369832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.752693+0000","last_change":"2024-09-13T14:19:32.403218+0000","last_active":"2024-09-13T14:19:37.752693+0000","last_peered":"2024-09-13T14:19:37.752693+0000","last_clean":"2024-09-13T14:19:37.752693+0000","last_became_active":"2024-09-13T14:19:27.913892+0000","last_became_peered":"2024-09-13T14:19:27.913892+0000","last_unstale":"2024-09-13T14:19:37.752693+0000","last_undegraded":"2024-09-13T14:19:37.752693+0000","last_fullsized":"2024-09-13T14:19:37.752693+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:49.778232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.995037+0000","last_change":"2024-09-13T14:19:32.223658+0000","last_active":"2024-09-13T14:19:37.995037+0000","last_peered":"2024-09-13T14:19:37.995037+0000","last_clean":"2024-09-13T14:19:37.995037+0000","last_became_active":"2024-09-13T14:19:27.912642+0000","last_became_peered":"2024-09-13T14:19:27.912642+0000","last_unstale":"2024-09-13T14:19:37.995037+0000","last_undegraded":"2024-09-13T14:19:37.995037+0000","last_fullsized":"2024-09-13T14:19:37.995037+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:11.887140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":263,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:19:32.402932+0000","last_change":"2024-09-13T14:18:58.652200+0000","last_active":"2024-09-13T14:19:32.402932+0000","last_peered":"2024-09-13T14:19:32.402932+0000","last_clean":"2024-09-13T14:19:32.402932+0000","last_became_active":"2024-09-13T14:18:58.651837+0000","last_became_peered":"2024-09-13T14:18:58.651837+0000","last_unstale":"2024-09-13T14:19:32.402932+0000","last_undegraded":"2024-09-13T14:19:32.402932+0000","last_fullsized":"2024-09-13T14:19:32.402932+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:10.115083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.46500000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.502}]},{"osd":3,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.58499999999999996}]},{"osd":4,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.57399999999999995}]},{"osd":5,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.55900000000000005}]},{"osd":6,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.61599999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495642,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":873480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.495}]},{"osd":1,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.377}]},{"osd":2,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.45200000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.48099999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"last":0.40200000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.495,"5min":0.495,"15min":0.495},"last":0.623},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.56999999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:19:57 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.64600000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757280,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.56000000000000005,"15min":0.57099999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65500000000000003,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52200000000000002,"15min":0.51800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68200000000000005,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.64400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69399999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.60199999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55800000000000005}]},{"osd":3,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.69599999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.61799999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.39400000000000002,"15min":0.38600000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.39600000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:19:31 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.38},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51900000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:19:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.44},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.45600000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018916,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.5},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.54300000000000004,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.53100000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.622},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.40200000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53500000000000003,"15min":0.53400000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.56699999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.57899999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:20:10 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36099999999999999},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.44,"5min":0.35699999999999998,"15min":0.34300000000000003},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.41999999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:19:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"last":0.44},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.63700000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.51400000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:28 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.35999999999999999,"15min":0.36399999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.46899999999999997,"5min":0.623,"15min":0.623},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.5}]},{"osd":1,"last update":"Fri Sep 13 14:19:28 2024","interfaces":[{"interface":"back","average":{"1min":0.34200000000000003,"5min":0.35799999999999998,"15min":0.36099999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.40100000000000002,"15min":0.39800000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.46500000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:19:28 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.47499999999999998,"15min":0.47199999999999998},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.66300000000000003,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.42199999999999999,"15min":0.42299999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:19:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:20:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.748,"5min":0.75,"15min":0.75},"last":1.1399999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.76300000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:20:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":1.101}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31768,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713384,"statfs":{"total":95995035648,"available":95962505216,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.36399999999999999,"15min":0.34799999999999998},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.58099999999999996,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.498},{"interface":"front","average":{"1min":0.377,"5min":0.38,"15min":0.35999999999999999},"min":{"1min":0.246,"5min":0.183,"15min":0.183},"max":{"1min":0.55600000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.308}]},{"osd":1,"last update":"Fri Sep 13 14:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.36899999999999999},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.55100000000000005,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40200000000000002,"15min":0.38400000000000001},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.63100000000000001,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.34399999999999997}]},{"osd":3,"last update":"Fri Sep 13 14:20:27 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46300000000000002,"15min":0.46899999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.60499999999999998,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.376},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.47199999999999998,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61199999999999999,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.441}]},{"osd":4,"last update":"Fri Sep 13 14:19:46 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.63200000000000001,"15min":0.63300000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.71799999999999997,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.624,"15min":0.623},"min":{"1min":0.47799999999999998,"5min":0.434,"15min":0.434},"max":{"1min":0.82499999999999996,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.54000000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:20:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58699999999999997,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.60999999999999999,"15min":0.61299999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.66900000000000004}]},{"osd":6,"last update":"Fri Sep 13 14:19:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.56699999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:19:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58199999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27672,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717480,"statfs":{"total":95995035648,"available":95966699520,"internally_reserved":0,"allocated":667648,"data_stored":282566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:44 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.38300000000000001,"15min":0.35299999999999998},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58299999999999996,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.371},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.315}]},{"osd":2,"last update":"Fri Sep 13 14:20:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.41599999999999998,"15min":0.41799999999999998},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58099999999999996,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.43099999999999999,"15min":0.432},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.57199999999999995,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.34300000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:20:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.43099999999999999,"15min":0.44},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.437,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64300000000000002,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.39600000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.39900000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61899999999999999,"15min":0.62},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.79000000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68700000000000006}]},{"osd":5,"last update":"Fri Sep 13 14:20:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78000000000000003,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.53300000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.66600000000000004}]},{"osd":7,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.75},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.71899999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:19:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.33000000000000002,"15min":0.312},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.32600000000000001},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.309}]},{"osd":2,"last update":"Fri Sep 13 14:20:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38,"15min":0.38500000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.38,"5min":0.40799999999999997,"15min":0.41499999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.55400000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.34000000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:20:29 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.44700000000000001,"15min":0.45200000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66700000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.432,"15min":0.434},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72599999999999998,"5min":0.876,"15min":0.876},"last":0.36399999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:19:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.65500000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.65400000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69999999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56299999999999994},"min":{"1min":0.38900000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.64000000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.749},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73699999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.70999999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:20:33.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:20:33.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:20:33] "GET /metrics HTTP/1.1" 200 28505 "" "Prometheus/2.43.0" 2024-09-13T14:20:33.757 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:33 smithi042 ceph-mon[29349]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:33.757 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:33 smithi042 ceph-mon[40707]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:33 smithi159 ceph-mon[33330]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:34.897 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:34 smithi042 ceph-mon[40707]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:34.898 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:34 smithi042 ceph-mon[29349]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:34.964 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:34.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:34 smithi159 ceph-mon[33330]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:35.209 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:34 smithi042 systemd[1]: Stopping Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:35.598 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:35 smithi042 podman[76678]: 2024-09-13 14:20:35.459026265 +0000 UTC m=+0.482920484 container died bf76ac51359892a8a3a739adedce63fdf12b1cb0293221781a0eeac16ab23345 (image=quay.ceph.io/ceph-ci/ceph:reef, name=zen_shannon, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T14:20:36.098 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[71506]: ts=2024-09-13T14:20:35.815Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:20:36.541 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:36 smithi042 podman[76756]: 2024-09-13 14:20:36.265111889 +0000 UTC m=+0.575192216 container died 072a94db452c714226738d49a8f430dc0715f79dc0ab92bbff902f0515937fa3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:36 smithi042 ceph-mon[29349]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:36 smithi042 ceph-mon[40707]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:36.848 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:36 smithi042 podman[76756]: 2024-09-13 14:20:36.647958392 +0000 UTC m=+0.958038717 container cleanup 072a94db452c714226738d49a8f430dc0715f79dc0ab92bbff902f0515937fa3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:36.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:36 smithi042 bash[76756]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:20:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:36 smithi159 ceph-mon[33330]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:37.199 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:36 smithi042 podman[76780]: 2024-09-13 14:20:36.875129688 +0000 UTC m=+1.036988454 container remove 072a94db452c714226738d49a8f430dc0715f79dc0ab92bbff902f0515937fa3 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:37.199 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:36 smithi042 podman[76780]: 2024-09-13 14:20:36.950669574 +0000 UTC m=+1.112528295 volume remove a0a898a8bad2da5cd140d2dc0e45eac33e8977738145b8da728a070530636d8e 2024-09-13T14:20:37.634 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:20:37.635 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:20:37.848 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:37 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:20:37.848 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:37 smithi042 systemd[1]: Stopped Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:37.848 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:37 smithi042 systemd[1]: Starting Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:38.114 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:37 smithi042 podman[76950]: 2024-09-13 14:20:37.899619185 +0000 UTC m=+0.248949450 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-13T14:20:38.114 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 podman[76950]: 2024-09-13 14:20:38.022422355 +0000 UTC m=+0.371752613 volume create c833daae139db6c12c7c3a72056faa5a51ea30e98cfe42a199db44aabb49db8f 2024-09-13T14:20:38.115 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 podman[76950]: 2024-09-13 14:20:38.114441674 +0000 UTC m=+0.463771932 container create 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:38.741 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 podman[76950]: 2024-09-13 14:20:38.45930562 +0000 UTC m=+0.808635887 container init 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:38.741 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 podman[76950]: 2024-09-13 14:20:38.464723765 +0000 UTC m=+0.814054031 container start 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:38.741 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.478Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T14:20:38.741 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.478Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T14:20:38.741 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.480Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.42 port=9094 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.481Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.504Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.504Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.507Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:38.507Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 bash[76950]: 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad 2024-09-13T14:20:38.742 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:38 smithi042 systemd[1]: Started Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:38.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:38 smithi159 ceph-mon[33330]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:38 smithi159 ceph-mon[33330]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:38 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:38 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:39.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[29349]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:39.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[29349]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:39.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:39.007 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:39.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[40707]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:39.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[40707]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:20:39.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:39.007 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:38 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:39.008 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":46,"stamp":"2024-09-13T14:20:36.804763+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":227236,"kb_used_data":6988,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733980,"statfs":{"total":767960285184,"available":767727595520,"internally_reserved":0,"allocated":7155712,"data_stored":4035028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":225431097},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002434"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.995309+0000","last_change":"2024-09-13T14:19:32.224331+0000","last_active":"2024-09-13T14:19:37.995309+0000","last_peered":"2024-09-13T14:19:37.995309+0000","last_clean":"2024-09-13T14:19:37.995309+0000","last_became_active":"2024-09-13T14:19:27.914929+0000","last_became_peered":"2024-09-13T14:19:27.914929+0000","last_unstale":"2024-09-13T14:19:37.995309+0000","last_undegraded":"2024-09-13T14:19:37.995309+0000","last_fullsized":"2024-09-13T14:19:37.995309+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:47.216732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.739681+0000","last_change":"2024-09-13T14:19:32.224489+0000","last_active":"2024-09-13T14:19:37.739681+0000","last_peered":"2024-09-13T14:19:37.739681+0000","last_clean":"2024-09-13T14:19:37.739681+0000","last_became_active":"2024-09-13T14:19:28.303467+0000","last_became_peered":"2024-09-13T14:19:28.303467+0000","last_unstale":"2024-09-13T14:19:37.739681+0000","last_undegraded":"2024-09-13T14:19:37.739681+0000","last_fullsized":"2024-09-13T14:19:37.739681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:47.015688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032689699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.752693+0000","last_change":"2024-09-13T14:19:32.403220+0000","last_active":"2024-09-13T14:19:37.752693+0000","last_peered":"2024-09-13T14:19:37.752693+0000","last_clean":"2024-09-13T14:19:37.752693+0000","last_became_active":"2024-09-13T14:19:28.303245+0000","last_became_peered":"2024-09-13T14:19:28.303245+0000","last_unstale":"2024-09-13T14:19:37.752693+0000","last_undegraded":"2024-09-13T14:19:37.752693+0000","last_fullsized":"2024-09-13T14:19:37.752693+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:32.196419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.739665+0000","last_change":"2024-09-13T14:19:32.224472+0000","last_active":"2024-09-13T14:19:37.739665+0000","last_peered":"2024-09-13T14:19:37.739665+0000","last_clean":"2024-09-13T14:19:37.739665+0000","last_became_active":"2024-09-13T14:19:27.913130+0000","last_became_peered":"2024-09-13T14:19:27.913130+0000","last_unstale":"2024-09-13T14:19:37.739665+0000","last_undegraded":"2024-09-13T14:19:37.739665+0000","last_fullsized":"2024-09-13T14:19:37.739665+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:25.471342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":22,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.994998+0000","last_change":"2024-09-13T14:19:32.223637+0000","last_active":"2024-09-13T14:19:37.994998+0000","last_peered":"2024-09-13T14:19:37.994998+0000","last_clean":"2024-09-13T14:19:37.994998+0000","last_became_active":"2024-09-13T14:19:27.912799+0000","last_became_peered":"2024-09-13T14:19:27.912799+0000","last_unstale":"2024-09-13T14:19:37.994998+0000","last_undegraded":"2024-09-13T14:19:37.994998+0000","last_fullsized":"2024-09-13T14:19:37.994998+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:07.658550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.651457+0000","last_change":"2024-09-13T14:19:32.219943+0000","last_active":"2024-09-13T14:19:37.651457+0000","last_peered":"2024-09-13T14:19:37.651457+0000","last_clean":"2024-09-13T14:19:37.651457+0000","last_became_active":"2024-09-13T14:19:28.274350+0000","last_became_peered":"2024-09-13T14:19:28.274350+0000","last_unstale":"2024-09-13T14:19:37.651457+0000","last_undegraded":"2024-09-13T14:19:37.651457+0000","last_fullsized":"2024-09-13T14:19:37.651457+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:22.369832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.752693+0000","last_change":"2024-09-13T14:19:32.403218+0000","last_active":"2024-09-13T14:19:37.752693+0000","last_peered":"2024-09-13T14:19:37.752693+0000","last_clean":"2024-09-13T14:19:37.752693+0000","last_became_active":"2024-09-13T14:19:27.913892+0000","last_became_peered":"2024-09-13T14:19:27.913892+0000","last_unstale":"2024-09-13T14:19:37.752693+0000","last_undegraded":"2024-09-13T14:19:37.752693+0000","last_fullsized":"2024-09-13T14:19:37.752693+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:49.778232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-13T14:19:37.995037+0000","last_change":"2024-09-13T14:19:32.223658+0000","last_active":"2024-09-13T14:19:37.995037+0000","last_peered":"2024-09-13T14:19:37.995037+0000","last_clean":"2024-09-13T14:19:37.995037+0000","last_became_active":"2024-09-13T14:19:27.912642+0000","last_became_peered":"2024-09-13T14:19:27.912642+0000","last_unstale":"2024-09-13T14:19:37.995037+0000","last_undegraded":"2024-09-13T14:19:37.995037+0000","last_fullsized":"2024-09-13T14:19:37.995037+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:11.887140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":263,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T14:19:32.402932+0000","last_change":"2024-09-13T14:18:58.652200+0000","last_active":"2024-09-13T14:19:32.402932+0000","last_peered":"2024-09-13T14:19:32.402932+0000","last_clean":"2024-09-13T14:19:32.402932+0000","last_became_active":"2024-09-13T14:18:58.651837+0000","last_became_peered":"2024-09-13T14:18:58.651837+0000","last_unstale":"2024-09-13T14:19:32.402932+0000","last_undegraded":"2024-09-13T14:19:32.402932+0000","last_fullsized":"2024-09-13T14:19:32.402932+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:10.115083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28252,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716900,"statfs":{"total":95995035648,"available":95966105600,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.45000000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.504}]},{"osd":2,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.52600000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.48799999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.46300000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.39800000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.438},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.377}]}]},{"osd":6,"up_from":46,"seq":197568495643,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":873480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38900000000000001,"5min":0.309,"15min":0.309},"max":{"1min":0.67300000000000004,"5min":1.0600000000000001,"15min":1.0600000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.58199999999999996,"15min":0.58799999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.749,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.51600000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.72599999999999998,"5min":1.113,"15min":1.113},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.52800000000000002,"15min":0.52600000000000002},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.46200000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.52800000000000002,"15min":0.52100000000000002},"min":{"1min":0.42799999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.443},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.54800000000000004,"15min":0.54700000000000004},"min":{"1min":0.40899999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78800000000000003,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.47199999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55100000000000005,"15min":0.55000000000000004},"min":{"1min":0.44400000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.55200000000000005,"15min":0.55400000000000005},"min":{"1min":0.378,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.74399999999999999,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.40699999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40500000000000003,"15min":0.39700000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.32300000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.41399999999999998,"15min":0.40300000000000002},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.375},{"interface":"front","average":{"1min":0.501,"5min":0.43099999999999999,"15min":0.41899999999999998},"min":{"1min":0.38300000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.38300000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:19:57 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.496}]}]},{"osd":5,"up_from":38,"seq":163208757281,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":282781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.56000000000000005,"15min":0.57099999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65500000000000003,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.52200000000000002,"15min":0.51800000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68200000000000005,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.47099999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.499,"15min":0.497},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55800000000000005,"15min":0.56000000000000005},"min":{"1min":0.41399999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.69399999999999995,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56499999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54400000000000004,"15min":0.54800000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.51800000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.69599999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.495},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.57599999999999996}]},{"osd":4,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.441,"5min":0.39400000000000002,"15min":0.38600000000000001},"min":{"1min":0.27500000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.36899999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:20:34 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.438,"15min":0.43099999999999999},"min":{"1min":0.32800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.70299999999999996,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.498,"15min":0.48899999999999999},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.316},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.39100000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:19:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018917,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.496},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.54300000000000004,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.44500000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53500000000000003,"15min":0.53400000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.55900000000000005,"15min":0.56200000000000006},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.56499999999999995}]},{"osd":3,"last update":"Fri Sep 13 14:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54200000000000004,"15min":0.54400000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.54700000000000004}]},{"osd":5,"last update":"Fri Sep 13 14:20:10 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.36699999999999999,"15min":0.36099999999999999},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.627},{"interface":"front","average":{"1min":0.44,"5min":0.35699999999999998,"15min":0.34300000000000003},"min":{"1min":0.26700000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.66300000000000003}]},{"osd":6,"last update":"Fri Sep 13 14:20:33 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.38800000000000001,"15min":0.38300000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46300000000000002,"15min":0.45500000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.68300000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.64200000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182441,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27676,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717476,"statfs":{"total":95995035648,"available":95966695424,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":0.35499999999999998,"15min":0.36199999999999999},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.38300000000000001,"15min":0.38700000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.61599999999999999,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.34100000000000003}]},{"osd":1,"last update":"Fri Sep 13 14:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.35599999999999998,"15min":0.35999999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.52800000000000002,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.40300000000000002,"5min":0.40200000000000002,"15min":0.39800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38900000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:20:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.46800000000000003,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41899999999999998,"15min":0.42199999999999999},"min":{"1min":0.308,"5min":0.252,"15min":0.252},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.40999999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:19:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.93700000000000006},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":1.018}]},{"osd":5,"last update":"Fri Sep 13 14:20:15 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56299999999999994,"15min":0.56000000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":1.0409999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.748,"5min":0.75,"15min":0.75},"last":0.56100000000000005}]},{"osd":6,"last update":"Fri Sep 13 14:20:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":1.0700000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.58199999999999996,"15min":0.57399999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":1.0309999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:20:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":1.054},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.71199999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31768,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713384,"statfs":{"total":95995035648,"available":95962505216,"internally_reserved":0,"allocated":671744,"data_stored":283093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.36399999999999999,"15min":0.34799999999999998},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.58099999999999996,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.377,"5min":0.38,"15min":0.35999999999999999},"min":{"1min":0.246,"5min":0.183,"15min":0.183},"max":{"1min":0.55600000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.54500000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:20:07 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.38900000000000001,"15min":0.36899999999999999},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.55100000000000005,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40200000000000002,"15min":0.38400000000000001},"min":{"1min":0.30499999999999999,"5min":0.223,"15min":0.223},"max":{"1min":0.63100000000000001,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.433}]},{"osd":3,"last update":"Fri Sep 13 14:20:27 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46300000000000002,"15min":0.46899999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.60499999999999998,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.47199999999999998,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61199999999999999,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.53200000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:19:46 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.63200000000000001,"15min":0.63300000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.71799999999999997,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.624,"15min":0.623},"min":{"1min":0.47799999999999998,"5min":0.434,"15min":0.434},"max":{"1min":0.82499999999999996,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.85699999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:20:10 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.58699999999999997,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.60999999999999999,"15min":0.61299999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.75700000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:19:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.77500000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:19:56 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.749},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.82499999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27672,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717480,"statfs":{"total":95995035648,"available":95966699520,"internally_reserved":0,"allocated":667648,"data_stored":282566,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:19:44 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.38300000000000001,"15min":0.35299999999999998},"min":{"1min":0.27500000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58299999999999996,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.373},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.378,"15min":0.36899999999999999},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.313}]},{"osd":2,"last update":"Fri Sep 13 14:20:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.41599999999999998,"15min":0.41799999999999998},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58099999999999996,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.43099999999999999,"15min":0.432},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.57199999999999995,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.42199999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:20:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.43099999999999999,"15min":0.44},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.47899999999999998,"15min":0.48899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.64300000000000002,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41199999999999998}]},{"osd":4,"last update":"Fri Sep 13 14:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.39900000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.79800000000000004,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61899999999999999,"15min":0.62},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.79000000000000004,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.65700000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:20:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56699999999999995},"min":{"1min":0.42999999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.78000000000000003,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.57099999999999995}]},{"osd":6,"last update":"Fri Sep 13 14:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.55900000000000005}]},{"osd":7,"last update":"Fri Sep 13 14:19:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.63300000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":873461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:19:42 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.33000000000000002,"15min":0.312},"min":{"1min":0.23599999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.34200000000000003,"15min":0.32600000000000001},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.42499999999999999}]},{"osd":2,"last update":"Fri Sep 13 14:20:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.38,"15min":0.38500000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.375},{"interface":"front","average":{"1min":0.38,"5min":0.40799999999999997,"15min":0.41499999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.55400000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.47999999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:20:29 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.44700000000000001,"15min":0.45200000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66700000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.432,"15min":0.434},"min":{"1min":0.308,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.72599999999999998,"5min":0.876,"15min":0.876},"last":0.44700000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:19:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57799999999999996,"15min":0.57999999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.65500000000000003,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.81399999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56699999999999995,"15min":0.56899999999999995},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.76700000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:20:14 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.53100000000000003,"15min":0.52900000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69999999999999996,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56299999999999994},"min":{"1min":0.38900000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.74399999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:20:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.58199999999999996,"15min":0.57799999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.61799999999999999,"15min":0.61599999999999999},"min":{"1min":0.44,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79200000000000004,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73199999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:19:59 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.77500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:20:39.010 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:20:39.010 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:20:39.010 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:20:39.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:20:39.216 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:39 smithi042 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:20:39.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:39 smithi042 ceph-mon[40707]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:20:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:39 smithi159 ceph-mon[33330]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:20:40.668 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:40.481Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000131448s 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[29349]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[40707]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[40707]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:20:40.949 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:40 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:40 smithi159 ceph-mon[33330]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:40 smithi159 ceph-mon[33330]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:20:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:40 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:41.138 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:20:41.138 INFO:teuthology.orchestra.run.smithi042.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:20:41.884 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:20:41.884 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T14:20:41.885 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T14:20:41.894 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T14:20:41.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-13T14:20:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:41 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1398778128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:20:41.999 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:41 smithi159 systemd[1]: Stopping Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:41 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1398778128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:20:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:41 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1398778128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:20:42.104 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.153Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:20:42.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[62576]: ts=2024-09-13T14:20:42.154Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:20:42.446 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 podman[65943]: 2024-09-13 14:20:42.164160031 +0000 UTC m=+0.111584135 container died a615bf639f4bc6b4a73a94bf905305815fdadc9435f96c1ae84ac2e4df4764c6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:42.748 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 podman[65943]: 2024-09-13 14:20:42.568344553 +0000 UTC m=+0.515768658 container cleanup a615bf639f4bc6b4a73a94bf905305815fdadc9435f96c1ae84ac2e4df4764c6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:42.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 bash[65943]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:20:42.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:42 smithi042 ceph-mon[29349]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:42 smithi042 ceph-mon[40707]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:43.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:42 smithi159 ceph-mon[33330]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:43.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:42 smithi159 podman[65960]: 2024-09-13 14:20:42.777801021 +0000 UTC m=+0.615593473 container remove a615bf639f4bc6b4a73a94bf905305815fdadc9435f96c1ae84ac2e4df4764c6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 systemd[1]: Stopped Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.173s CPU time. 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 systemd[1]: Starting Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 podman[66059]: 2024-09-13 14:20:43.113859709 +0000 UTC m=+0.017143350 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:20:43.324 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 podman[66059]: 2024-09-13 14:20:43.212204804 +0000 UTC m=+0.115488439 container create 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:43.631 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Bus STOPPING 2024-09-13T14:20:43.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 podman[66059]: 2024-09-13 14:20:43.431084561 +0000 UTC m=+0.334368196 container init 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:43.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 podman[66059]: 2024-09-13 14:20:43.436282883 +0000 UTC m=+0.339566522 container start 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.459Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.459Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.459Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi159 (none))" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.459Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.459Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.461Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.461Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.463Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.463Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.465Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:20:43.750 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.465Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.469µs 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.465Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.466Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.466Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.466Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.602µs wal_replay_duration=1.240141ms wbl_replay_duration=314ns total_replay_duration=1.276339ms 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.467Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.467Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.467Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.486Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.948124ms db_storage=769ns remote_storage=1.492µs web_handler=376ns query_engine=696ns scrape=387.683µs scrape_sd=93.019µs notify=10.799µs notify_sd=16.909µs rules=18.126486ms tracing=5.823µs 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.486Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:20:43.486Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 bash[66059]: 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 2024-09-13T14:20:43.751 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:20:43 smithi159 systemd[1]: Started Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:20:44.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:20:44.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Bus STOPPED 2024-09-13T14:20:44.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Bus STARTING 2024-09-13T14:20:44.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Serving on http://:::9283 2024-09-13T14:20:44.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Bus STARTED 2024-09-13T14:20:44.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:43] ENGINE Bus STOPPING 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STOPPED 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STARTING 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Serving on http://:::9283 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STARTED 2024-09-13T14:20:44.539 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STOPPING 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:44.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:44 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi159.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi159.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:44.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T14:20:45.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:20:45.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STOPPED 2024-09-13T14:20:45.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STARTING 2024-09-13T14:20:45.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Serving on http://:::9283 2024-09-13T14:20:45.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:20:44] ENGINE Bus STARTED 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[40707]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[40707]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[29349]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:20:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[29349]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:20:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[29326]: 2024-09-13T14:20:45.771+0000 7fe4e02d5640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:20:46.123 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:45 smithi159 ceph-mon[33330]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:20:46.123 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:45 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T14:20:46.123 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:45 smithi159 ceph-mon[33330]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T14:20:46.123 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:45 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.124 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:45 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:20:46.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:20:47.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: fsmap foo:0 2024-09-13T14:20:47.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: Saving service mds.foo spec with placement count:2 2024-09-13T14:20:47.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:46 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:20:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: fsmap foo:0 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: Saving service mds.foo spec with placement count:2 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:47.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: fsmap foo:0 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: Saving service mds.foo spec with placement count:2 2024-09-13T14:20:47.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:46 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:47.202 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.072 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:48.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:47 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:48.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:47 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:48.786 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:20:48.484Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003480613s 2024-09-13T14:20:49.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[40707]: pgmap v55: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:49.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[40707]: Deploying daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:20:49.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[40707]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:20:49.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[40707]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:20:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[29349]: pgmap v55: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[29349]: Deploying daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:20:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:20:49.042 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:48 smithi042 ceph-mon[29349]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:20:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:48 smithi159 ceph-mon[33330]: pgmap v55: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:48 smithi159 ceph-mon[33330]: Deploying daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:20:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:48 smithi159 ceph-mon[33330]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:20:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:48 smithi159 ceph-mon[33330]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T14:20:50.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T14:20:50.928 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:51.040 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:50 smithi042 ceph-mon[29349]: pgmap v57: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:51.040 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:50 smithi042 ceph-mon[40707]: pgmap v57: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:50 smithi159 ceph-mon[33330]: pgmap v57: 73 pgs: 5 creating+activating, 26 creating+peering, 42 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:52.905 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:52 smithi042 ceph-mon[29349]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:52.905 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:52 smithi042 ceph-mon[40707]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:53.245 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:20:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:20:53] "GET /metrics HTTP/1.1" 200 28492 "" "Prometheus/2.43.0" 2024-09-13T14:20:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:52 smithi159 ceph-mon[33330]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:53.806 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:20:53.815 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:20:53.815 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:20:53.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:20:54.110 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:55.088 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:55.088 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.088 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.088 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.088 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.089 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.089 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:55.089 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:54 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T14:20:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:54 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:20:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: Deploying daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:boot 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: daemon mds.foo.smithi042.pkkbnw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: Cluster is now healthy 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: fsmap foo:0 1 up:standby 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:creating} 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: daemon mds.foo.smithi042.pkkbnw is now active in filesystem foo as rank 0 2024-09-13T14:20:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:56.008 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:20:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: Deploying daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:20:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:boot 2024-09-13T14:20:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: daemon mds.foo.smithi042.pkkbnw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:20:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: Cluster is now healthy 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: fsmap foo:0 1 up:standby 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:creating} 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: daemon mds.foo.smithi042.pkkbnw is now active in filesystem foo as rank 0 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: Deploying daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:boot 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: daemon mds.foo.smithi042.pkkbnw assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T14:20:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: Cluster is now healthy 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: fsmap foo:0 1 up:standby 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:creating} 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: daemon mds.foo.smithi042.pkkbnw is now active in filesystem foo as rank 0 2024-09-13T14:20:56.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:20:56.806 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":21,"flags":0,"active_gid":14442,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6800","nonce":3689509320},{"type":"v1","addr":"172.21.15.42:6801","nonce":3689509320}]},"active_addr":"172.21.15.42:6801/3689509320","active_change":"2024-09-13T14:19:24.781610+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24349,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":471235645}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":800406081}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":1962929896}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":3559778733}]}]} 2024-09-13T14:20:56.810 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:20:56.810 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:20:56.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:20:57.040 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/167081168' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/167081168' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:56 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:20:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:20:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 2024-09-13T14:20:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/167081168' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T14:20:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:20:57.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:56 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:20:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[29349]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[29349]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:boot 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[40707]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[40707]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:boot 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:20:58.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:57 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:58.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:57 smithi159 ceph-mon[33330]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T14:20:58.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:57 smithi159 ceph-mon[33330]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:boot 2024-09-13T14:20:58.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:57 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:58.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:57 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:20:58.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:57 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:59.749 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:20:59.749 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":71,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:20:47.777835+0000","last_up_change":"2024-09-13T14:18:55.210301+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6803","nonce":917582203}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6805","nonce":917582203}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6809","nonce":917582203}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6807","nonce":917582203}]},"public_addr":"172.21.15.42:6803/917582203","cluster_addr":"172.21.15.42:6805/917582203","heartbeat_back_addr":"172.21.15.42:6809/917582203","heartbeat_front_addr":"172.21.15.42:6807/917582203","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6811","nonce":3832896807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6813","nonce":3832896807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6817","nonce":3832896807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6815","nonce":3832896807}]},"public_addr":"172.21.15.42:6811/3832896807","cluster_addr":"172.21.15.42:6813/3832896807","heartbeat_back_addr":"172.21.15.42:6817/3832896807","heartbeat_front_addr":"172.21.15.42:6815/3832896807","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6819","nonce":2844866888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6821","nonce":2844866888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6825","nonce":2844866888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6823","nonce":2844866888}]},"public_addr":"172.21.15.42:6819/2844866888","cluster_addr":"172.21.15.42:6821/2844866888","heartbeat_back_addr":"172.21.15.42:6825/2844866888","heartbeat_front_addr":"172.21.15.42:6823/2844866888","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6827","nonce":38876887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6829","nonce":38876887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6833","nonce":38876887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6831","nonce":38876887}]},"public_addr":"172.21.15.42:6827/38876887","cluster_addr":"172.21.15.42:6829/38876887","heartbeat_back_addr":"172.21.15.42:6833/38876887","heartbeat_front_addr":"172.21.15.42:6831/38876887","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:20:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:59 smithi159 ceph-mon[33330]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-09-13T14:20:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:59 smithi159 ceph-mon[33330]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:20:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:59 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:20:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:20:59 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/622948644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[29349]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[29349]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/622948644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[40707]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[40707]: mds.? [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] up:active 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:21:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:20:59 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/622948644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:00.624 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:21:00.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:21:01.164 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:00 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:00 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:00 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:01.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:00 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[40707]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[40707]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:standby 2024-09-13T14:21:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:21:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[29349]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[29349]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:standby 2024-09-13T14:21:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:02 smithi042 ceph-mon[29349]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:21:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:02 smithi159 ceph-mon[33330]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:02 smithi159 ceph-mon[33330]: mds.? [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] up:standby 2024-09-13T14:21:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:02 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:21:03.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:03] "GET /metrics HTTP/1.1" 200 33414 "" "Prometheus/2.43.0" 2024-09-13T14:21:03.833 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:21:03.833 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":71,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:20:47.777835+0000","last_up_change":"2024-09-13T14:18:55.210301+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6803","nonce":917582203}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6805","nonce":917582203}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6809","nonce":917582203}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":917582203},{"type":"v1","addr":"172.21.15.42:6807","nonce":917582203}]},"public_addr":"172.21.15.42:6803/917582203","cluster_addr":"172.21.15.42:6805/917582203","heartbeat_back_addr":"172.21.15.42:6809/917582203","heartbeat_front_addr":"172.21.15.42:6807/917582203","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6811","nonce":3832896807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6813","nonce":3832896807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6817","nonce":3832896807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":3832896807},{"type":"v1","addr":"172.21.15.42:6815","nonce":3832896807}]},"public_addr":"172.21.15.42:6811/3832896807","cluster_addr":"172.21.15.42:6813/3832896807","heartbeat_back_addr":"172.21.15.42:6817/3832896807","heartbeat_front_addr":"172.21.15.42:6815/3832896807","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6819","nonce":2844866888}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6821","nonce":2844866888}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6825","nonce":2844866888}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":2844866888},{"type":"v1","addr":"172.21.15.42:6823","nonce":2844866888}]},"public_addr":"172.21.15.42:6819/2844866888","cluster_addr":"172.21.15.42:6821/2844866888","heartbeat_back_addr":"172.21.15.42:6825/2844866888","heartbeat_front_addr":"172.21.15.42:6823/2844866888","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6827","nonce":38876887}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6829","nonce":38876887}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6833","nonce":38876887}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":38876887},{"type":"v1","addr":"172.21.15.42:6831","nonce":38876887}]},"public_addr":"172.21.15.42:6827/38876887","cluster_addr":"172.21.15.42:6829/38876887","heartbeat_back_addr":"172.21.15.42:6833/38876887","heartbeat_front_addr":"172.21.15.42:6831/38876887","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":67,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:21:04.476 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4116118460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:04.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:04.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:04.478 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:04 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4116118460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:04.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:04.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:04 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4116118460' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T14:21:05.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:21:05.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:21:05.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:21:05.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:21:05.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:21:05.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:21:05.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:21:05.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:21:05.639 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:05.696 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:05.697 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:06.036 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:05 smithi042 ceph-mon[40707]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:06.036 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:05 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:06.036 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:05 smithi042 ceph-mon[29349]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:06.037 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:05 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:05 smithi159 ceph-mon[33330]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:05 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:07 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:07 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:07 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:07 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:07 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:07 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:08.228 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:08.228 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:08.239 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:08.240 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:08.244 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:08.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:08 smithi042 ceph-mon[29349]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:08.994 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:08 smithi042 ceph-mon[40707]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:08 smithi159 ceph-mon[33330]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-13T14:21:10.481 INFO:teuthology.orchestra.run.smithi042.stdout:197568495650 2024-09-13T14:21:10.481 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:21:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:10 smithi159 ceph-mon[33330]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:10 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:11.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:10 smithi042 ceph-mon[40707]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:11.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:10 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:11.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:10 smithi042 ceph-mon[29349]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:11.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:10 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:12.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:12 smithi042 ceph-mon[29349]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:12.995 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:12 smithi042 ceph-mon[40707]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:12.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:12 smithi159 ceph-mon[33330]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-13T14:21:13.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:13] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-13T14:21:14.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:14 smithi159 ceph-mon[33330]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:14 smithi042 ceph-mon[29349]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:14 smithi042 ceph-mon[40707]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:16 smithi042 ceph-mon[29349]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:16 smithi042 ceph-mon[40707]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:17.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:16 smithi159 ceph-mon[33330]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:18.177 INFO:teuthology.orchestra.run.smithi042.stdout:77309411383 2024-09-13T14:21:18.177 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.718 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:18.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:18.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:18.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:18 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:18 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:19.790 INFO:teuthology.orchestra.run.smithi042.stdout:55834574907 2024-09-13T14:21:19.790 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:21:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:20 smithi042 ceph-mon[29349]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:20 smithi042 ceph-mon[40707]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:20.730 INFO:teuthology.orchestra.run.smithi042.stdout:128849018926 2024-09-13T14:21:20.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:21:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:20 smithi159 ceph-mon[33330]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:21.726 INFO:teuthology.orchestra.run.smithi042.stdout:231928234015 2024-09-13T14:21:21.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:21:22.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:22 smithi042 ceph-mon[29349]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:22 smithi042 ceph-mon[40707]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:22.872 INFO:teuthology.orchestra.run.smithi042.stdout:163208757289 2024-09-13T14:21:22.873 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:21:22.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:22 smithi159 ceph-mon[33330]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:23.163 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:23] "GET /metrics HTTP/1.1" 200 33701 "" "Prometheus/2.43.0" 2024-09-13T14:21:23.649 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:23.649 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:23.649 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:24.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:24 smithi042 ceph-mon[29349]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:24 smithi042 ceph-mon[40707]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:24 smithi159 ceph-mon[33330]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:25.502 INFO:teuthology.orchestra.run.smithi042.stdout:34359738434 2024-09-13T14:21:25.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:21:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]: dispatch 2024-09-13T14:21:25.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-13T14:21:25.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:26.688 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[40707]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:26.688 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T14:21:26.688 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T14:21:26.688 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[40707]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[29349]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:21:26.689 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:26 smithi042 ceph-mon[29349]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:21:26.710 INFO:teuthology.orchestra.run.smithi042.stdout:107374182451 2024-09-13T14:21:26.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:21:26.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:26 smithi159 ceph-mon[33330]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:26 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T14:21:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:26 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T14:21:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:26 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:21:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:26 smithi159 ceph-mon[33330]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T14:21:27.449 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:27.450 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:27.526 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:27 smithi042 ceph-mon[29349]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:21:27.845 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:27 smithi042 ceph-mon[40707]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:21:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:27 smithi159 ceph-mon[33330]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T14:21:28.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:28 smithi042 ceph-mon[29349]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:28.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:28 smithi042 ceph-mon[40707]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:28.796 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:28 smithi159 ceph-mon[33330]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:30.609 INFO:teuthology.orchestra.run.smithi042.stdout:77309411385 2024-09-13T14:21:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:30 smithi042 ceph-mon[29349]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:30 smithi042 ceph-mon[40707]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:30 smithi159 ceph-mon[33330]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:31.561 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411385 for osd.2 2024-09-13T14:21:31.561 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:31.803 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:31 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/69161554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:21:31.803 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:31 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/69161554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:21:31.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:31 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/69161554' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T14:21:32.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:32 smithi042 ceph-mon[29349]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:32 smithi042 ceph-mon[40707]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:32.900 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:32 smithi159 ceph-mon[33330]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:33.238 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:33] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.43.0" 2024-09-13T14:21:33.609 INFO:teuthology.orchestra.run.smithi042.stdout:197568495655 2024-09-13T14:21:34.689 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:34 smithi042 ceph-mon[40707]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:34.689 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:34 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4048167823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:21:34.690 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:34 smithi042 ceph-mon[29349]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:34.690 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:34 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4048167823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:21:34.874 INFO:teuthology.orchestra.run.smithi042.stdout:231928234017 2024-09-13T14:21:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:34 smithi159 ceph-mon[33330]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:34 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4048167823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T14:21:35.630 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234015 got 231928234017 for osd.7 2024-09-13T14:21:35.630 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:35.803 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:35 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1607890074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:21:35.803 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:35 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1607890074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:21:35.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:35 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1607890074' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T14:21:36.267 INFO:teuthology.orchestra.run.smithi042.stdout:128849018929 2024-09-13T14:21:36.587 INFO:teuthology.orchestra.run.smithi042.stdout:163208757293 2024-09-13T14:21:36.725 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:36 smithi042 ceph-mon[40707]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:36.725 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:36 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2909493627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:21:36.725 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:36 smithi042 ceph-mon[29349]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:36.725 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:36 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2909493627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:21:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:36 smithi159 ceph-mon[33330]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:36 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2909493627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T14:21:37.122 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018926 got 128849018929 for osd.4 2024-09-13T14:21:37.122 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:37.470 INFO:teuthology.orchestra.run.smithi042.stdout:55834574910 2024-09-13T14:21:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:37 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4126311603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:21:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:37 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1195520338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:21:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:37 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4126311603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:21:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:37 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1195520338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:21:37.844 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495650 got 197568495655 for osd.6 2024-09-13T14:21:37.845 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:37 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4126311603' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T14:21:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:37 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1195520338' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T14:21:38.477 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757289 got 163208757293 for osd.5 2024-09-13T14:21:38.478 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:38.838 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:38 smithi042 ceph-mon[29349]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:38.839 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:38 smithi042 ceph-mon[40707]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:38.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:38 smithi159 ceph-mon[33330]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:39.140 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574910 for osd.1 2024-09-13T14:21:39.140 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:39.147 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:39.147 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:40 smithi042 ceph-mon[29349]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:40 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:40 smithi042 ceph-mon[40707]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:40 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:40 smithi159 ceph-mon[33330]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:40 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:42.153 INFO:teuthology.orchestra.run.smithi042.stdout:34359738438 2024-09-13T14:21:42.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:42 smithi042 ceph-mon[40707]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:42.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:42 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3432332039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:21:42.824 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:42 smithi042 ceph-mon[29349]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:42.824 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:42 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3432332039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:21:42.846 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738434 got 34359738438 for osd.0 2024-09-13T14:21:42.846 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:42.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:42 smithi159 ceph-mon[33330]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:42.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:42 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3432332039' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T14:21:43.228 INFO:teuthology.orchestra.run.smithi042.stdout:107374182455 2024-09-13T14:21:43.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:43] "GET /metrics HTTP/1.1" 200 33713 "" "Prometheus/2.43.0" 2024-09-13T14:21:43.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:43 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/258421264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:21:43.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:43 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/258421264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:21:43.882 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182451 got 107374182455 for osd.3 2024-09-13T14:21:43.882 DEBUG:teuthology.parallel:result is None 2024-09-13T14:21:43.882 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:21:43.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:21:43.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:43 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/258421264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T14:21:44.095 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:44 smithi042 ceph-mon[29349]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:44.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:44 smithi042 ceph-mon[40707]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:44.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:44 smithi159 ceph-mon[33330]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:46.037 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:21:46.038 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:21:46.756 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:46 smithi042 ceph-mon[40707]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:46.757 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:46 smithi042 ceph-mon[29349]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:46.784 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-13T14:21:44.818957+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":228020,"kb_used_data":7756,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733196,"statfs":{"total":767960285184,"available":767726792704,"internally_reserved":0,"allocated":7942144,"data_stored":4446138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":225431097},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003142"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522131+0000","last_change":"2024-09-13T14:20:46.791991+0000","last_active":"2024-09-13T14:21:25.522131+0000","last_peered":"2024-09-13T14:21:25.522131+0000","last_clean":"2024-09-13T14:21:25.522131+0000","last_became_active":"2024-09-13T14:20:46.791711+0000","last_became_peered":"2024-09-13T14:20:46.791711+0000","last_unstale":"2024-09-13T14:21:25.522131+0000","last_undegraded":"2024-09-13T14:21:25.522131+0000","last_fullsized":"2024-09-13T14:21:25.522131+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:18.711586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524163+0000","last_change":"2024-09-13T14:20:45.776612+0000","last_active":"2024-09-13T14:21:25.524163+0000","last_peered":"2024-09-13T14:21:25.524163+0000","last_clean":"2024-09-13T14:21:25.524163+0000","last_became_active":"2024-09-13T14:20:45.776322+0000","last_became_peered":"2024-09-13T14:20:45.776322+0000","last_unstale":"2024-09-13T14:21:25.524163+0000","last_undegraded":"2024-09-13T14:21:25.524163+0000","last_fullsized":"2024-09-13T14:21:25.524163+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:02.371140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521199+0000","last_change":"2024-09-13T14:20:46.006969+0000","last_active":"2024-09-13T14:21:25.521199+0000","last_peered":"2024-09-13T14:21:25.521199+0000","last_clean":"2024-09-13T14:21:25.521199+0000","last_became_active":"2024-09-13T14:20:46.006624+0000","last_became_peered":"2024-09-13T14:20:46.006624+0000","last_unstale":"2024-09-13T14:21:25.521199+0000","last_undegraded":"2024-09-13T14:21:25.521199+0000","last_fullsized":"2024-09-13T14:21:25.521199+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:30.352892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516249+0000","last_change":"2024-09-13T14:20:45.790720+0000","last_active":"2024-09-13T14:21:25.516249+0000","last_peered":"2024-09-13T14:21:25.516249+0000","last_clean":"2024-09-13T14:21:25.516249+0000","last_became_active":"2024-09-13T14:20:45.790417+0000","last_became_peered":"2024-09-13T14:20:45.790417+0000","last_unstale":"2024-09-13T14:21:25.516249+0000","last_undegraded":"2024-09-13T14:21:25.516249+0000","last_fullsized":"2024-09-13T14:21:25.516249+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:59.219408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516773+0000","last_change":"2024-09-13T14:20:46.789778+0000","last_active":"2024-09-13T14:21:25.516773+0000","last_peered":"2024-09-13T14:21:25.516773+0000","last_clean":"2024-09-13T14:21:25.516773+0000","last_became_active":"2024-09-13T14:20:46.789470+0000","last_became_peered":"2024-09-13T14:20:46.789470+0000","last_unstale":"2024-09-13T14:21:25.516773+0000","last_undegraded":"2024-09-13T14:21:25.516773+0000","last_fullsized":"2024-09-13T14:21:25.516773+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:45.169563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524012+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:21:25.524012+0000","last_peered":"2024-09-13T14:21:25.524012+0000","last_clean":"2024-09-13T14:21:25.524012+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:21:25.524012+0000","last_undegraded":"2024-09-13T14:21:25.524012+0000","last_fullsized":"2024-09-13T14:21:25.524012+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.535073+0000","last_change":"2024-09-13T14:21:26.535073+0000","last_active":"2024-09-13T14:21:26.535073+0000","last_peered":"2024-09-13T14:21:26.535073+0000","last_clean":"2024-09-13T14:21:26.535073+0000","last_became_active":"2024-09-13T14:21:26.534784+0000","last_became_peered":"2024-09-13T14:21:26.534784+0000","last_unstale":"2024-09-13T14:21:26.535073+0000","last_undegraded":"2024-09-13T14:21:26.535073+0000","last_fullsized":"2024-09-13T14:21:26.535073+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:57.054086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521070+0000","last_change":"2024-09-13T14:20:45.777470+0000","last_active":"2024-09-13T14:21:25.521070+0000","last_peered":"2024-09-13T14:21:25.521070+0000","last_clean":"2024-09-13T14:21:25.521070+0000","last_became_active":"2024-09-13T14:20:45.777201+0000","last_became_peered":"2024-09-13T14:20:45.777201+0000","last_unstale":"2024-09-13T14:21:25.521070+0000","last_undegraded":"2024-09-13T14:21:25.521070+0000","last_fullsized":"2024-09-13T14:21:25.521070+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:16.705041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524501+0000","last_change":"2024-09-13T14:20:45.785014+0000","last_active":"2024-09-13T14:21:25.524501+0000","last_peered":"2024-09-13T14:21:25.524501+0000","last_clean":"2024-09-13T14:21:25.524501+0000","last_became_active":"2024-09-13T14:20:45.784696+0000","last_became_peered":"2024-09-13T14:20:45.784696+0000","last_unstale":"2024-09-13T14:21:25.524501+0000","last_undegraded":"2024-09-13T14:21:25.524501+0000","last_fullsized":"2024-09-13T14:21:25.524501+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:56.609957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520461+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:21:25.520461+0000","last_peered":"2024-09-13T14:21:25.520461+0000","last_clean":"2024-09-13T14:21:25.520461+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:21:25.520461+0000","last_undegraded":"2024-09-13T14:21:25.520461+0000","last_fullsized":"2024-09-13T14:21:25.520461+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524474+0000","last_change":"2024-09-13T14:20:45.774478+0000","last_active":"2024-09-13T14:21:25.524474+0000","last_peered":"2024-09-13T14:21:25.524474+0000","last_clean":"2024-09-13T14:21:25.524474+0000","last_became_active":"2024-09-13T14:20:45.774224+0000","last_became_peered":"2024-09-13T14:20:45.774224+0000","last_unstale":"2024-09-13T14:21:25.524474+0000","last_undegraded":"2024-09-13T14:21:25.524474+0000","last_fullsized":"2024-09-13T14:21:25.524474+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:07.600680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517802+0000","last_change":"2024-09-13T14:20:46.788964+0000","last_active":"2024-09-13T14:21:25.517802+0000","last_peered":"2024-09-13T14:21:25.517802+0000","last_clean":"2024-09-13T14:21:25.517802+0000","last_became_active":"2024-09-13T14:20:46.788690+0000","last_became_peered":"2024-09-13T14:20:46.788690+0000","last_unstale":"2024-09-13T14:21:25.517802+0000","last_undegraded":"2024-09-13T14:21:25.517802+0000","last_fullsized":"2024-09-13T14:21:25.517802+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.736138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524431+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:21:25.524431+0000","last_peered":"2024-09-13T14:21:25.524431+0000","last_clean":"2024-09-13T14:21:25.524431+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:21:25.524431+0000","last_undegraded":"2024-09-13T14:21:25.524431+0000","last_fullsized":"2024-09-13T14:21:25.524431+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520508+0000","last_change":"2024-09-13T14:20:46.007505+0000","last_active":"2024-09-13T14:21:25.520508+0000","last_peered":"2024-09-13T14:21:25.520508+0000","last_clean":"2024-09-13T14:21:25.520508+0000","last_became_active":"2024-09-13T14:20:46.007236+0000","last_became_peered":"2024-09-13T14:20:46.007236+0000","last_unstale":"2024-09-13T14:21:25.520508+0000","last_undegraded":"2024-09-13T14:21:25.520508+0000","last_fullsized":"2024-09-13T14:21:25.520508+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:49.388877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521019+0000","last_change":"2024-09-13T14:20:45.768232+0000","last_active":"2024-09-13T14:21:25.521019+0000","last_peered":"2024-09-13T14:21:25.521019+0000","last_clean":"2024-09-13T14:21:25.521019+0000","last_became_active":"2024-09-13T14:20:45.767969+0000","last_became_peered":"2024-09-13T14:20:45.767969+0000","last_unstale":"2024-09-13T14:21:25.521019+0000","last_undegraded":"2024-09-13T14:21:25.521019+0000","last_fullsized":"2024-09-13T14:21:25.521019+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:33.316679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516344+0000","last_change":"2024-09-13T14:20:46.786055+0000","last_active":"2024-09-13T14:21:25.516344+0000","last_peered":"2024-09-13T14:21:25.516344+0000","last_clean":"2024-09-13T14:21:25.516344+0000","last_became_active":"2024-09-13T14:20:46.785770+0000","last_became_peered":"2024-09-13T14:20:46.785770+0000","last_unstale":"2024-09-13T14:21:25.516344+0000","last_undegraded":"2024-09-13T14:21:25.516344+0000","last_fullsized":"2024-09-13T14:21:25.516344+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:00.273047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522088+0000","last_change":"2024-09-13T14:20:45.781925+0000","last_active":"2024-09-13T14:21:25.522088+0000","last_peered":"2024-09-13T14:21:25.522088+0000","last_clean":"2024-09-13T14:21:25.522088+0000","last_became_active":"2024-09-13T14:20:45.781628+0000","last_became_peered":"2024-09-13T14:20:45.781628+0000","last_unstale":"2024-09-13T14:21:25.522088+0000","last_undegraded":"2024-09-13T14:21:25.522088+0000","last_fullsized":"2024-09-13T14:21:25.522088+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:02.719731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518012+0000","last_change":"2024-09-13T14:20:46.806643+0000","last_active":"2024-09-13T14:21:25.518012+0000","last_peered":"2024-09-13T14:21:25.518012+0000","last_clean":"2024-09-13T14:21:25.518012+0000","last_became_active":"2024-09-13T14:20:46.806276+0000","last_became_peered":"2024-09-13T14:20:46.806276+0000","last_unstale":"2024-09-13T14:21:25.518012+0000","last_undegraded":"2024-09-13T14:21:25.518012+0000","last_fullsized":"2024-09-13T14:21:25.518012+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:06.830482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521036+0000","last_change":"2024-09-13T14:20:45.776337+0000","last_active":"2024-09-13T14:21:25.521036+0000","last_peered":"2024-09-13T14:21:25.521036+0000","last_clean":"2024-09-13T14:21:25.521036+0000","last_became_active":"2024-09-13T14:20:45.776009+0000","last_became_peered":"2024-09-13T14:20:45.776009+0000","last_unstale":"2024-09-13T14:21:25.521036+0000","last_undegraded":"2024-09-13T14:21:25.521036+0000","last_fullsized":"2024-09-13T14:21:25.521036+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:47.633177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518256+0000","last_change":"2024-09-13T14:20:46.791130+0000","last_active":"2024-09-13T14:21:25.518256+0000","last_peered":"2024-09-13T14:21:25.518256+0000","last_clean":"2024-09-13T14:21:25.518256+0000","last_became_active":"2024-09-13T14:20:46.790865+0000","last_became_peered":"2024-09-13T14:20:46.790865+0000","last_unstale":"2024-09-13T14:21:25.518256+0000","last_undegraded":"2024-09-13T14:21:25.518256+0000","last_fullsized":"2024-09-13T14:21:25.518256+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:01.653014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517986+0000","last_change":"2024-09-13T14:20:45.776700+0000","last_active":"2024-09-13T14:21:25.517986+0000","last_peered":"2024-09-13T14:21:25.517986+0000","last_clean":"2024-09-13T14:21:25.517986+0000","last_became_active":"2024-09-13T14:20:45.776221+0000","last_became_peered":"2024-09-13T14:20:45.776221+0000","last_unstale":"2024-09-13T14:21:25.517986+0000","last_undegraded":"2024-09-13T14:21:25.517986+0000","last_fullsized":"2024-09-13T14:21:25.517986+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:07.172380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.523446+0000","last_change":"2024-09-13T14:21:26.523446+0000","last_active":"2024-09-13T14:21:26.523446+0000","last_peered":"2024-09-13T14:21:26.523446+0000","last_clean":"2024-09-13T14:21:26.523446+0000","last_became_active":"2024-09-13T14:21:26.523104+0000","last_became_peered":"2024-09-13T14:21:26.523104+0000","last_unstale":"2024-09-13T14:21:26.523446+0000","last_undegraded":"2024-09-13T14:21:26.523446+0000","last_fullsized":"2024-09-13T14:21:26.523446+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:26.394962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522168+0000","last_change":"2024-09-13T14:20:45.771630+0000","last_active":"2024-09-13T14:21:25.522168+0000","last_peered":"2024-09-13T14:21:25.522168+0000","last_clean":"2024-09-13T14:21:25.522168+0000","last_became_active":"2024-09-13T14:20:45.771318+0000","last_became_peered":"2024-09-13T14:20:45.771318+0000","last_unstale":"2024-09-13T14:21:25.522168+0000","last_undegraded":"2024-09-13T14:21:25.522168+0000","last_fullsized":"2024-09-13T14:21:25.522168+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:14.815638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520577+0000","last_change":"2024-09-13T14:20:46.792737+0000","last_active":"2024-09-13T14:21:25.520577+0000","last_peered":"2024-09-13T14:21:25.520577+0000","last_clean":"2024-09-13T14:21:25.520577+0000","last_became_active":"2024-09-13T14:20:46.792474+0000","last_became_peered":"2024-09-13T14:20:46.792474+0000","last_unstale":"2024-09-13T14:21:25.520577+0000","last_undegraded":"2024-09-13T14:21:25.520577+0000","last_fullsized":"2024-09-13T14:21:25.520577+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:26:32.385951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524028+0000","last_change":"2024-09-13T14:18:58.652200+0000","last_active":"2024-09-13T14:21:25.524028+0000","last_peered":"2024-09-13T14:21:25.524028+0000","last_clean":"2024-09-13T14:21:25.524028+0000","last_became_active":"2024-09-13T14:18:58.651837+0000","last_became_peered":"2024-09-13T14:18:58.651837+0000","last_unstale":"2024-09-13T14:21:25.524028+0000","last_undegraded":"2024-09-13T14:21:25.524028+0000","last_fullsized":"2024-09-13T14:21:25.524028+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:10.115083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.551415+0000","last_change":"2024-09-13T14:19:32.223658+0000","last_active":"2024-09-13T14:21:26.551415+0000","last_peered":"2024-09-13T14:21:26.551415+0000","last_clean":"2024-09-13T14:21:26.551415+0000","last_became_active":"2024-09-13T14:19:27.912642+0000","last_became_peered":"2024-09-13T14:19:27.912642+0000","last_unstale":"2024-09-13T14:21:26.551415+0000","last_undegraded":"2024-09-13T14:21:26.551415+0000","last_fullsized":"2024-09-13T14:21:26.551415+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:11.887140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521997+0000","last_change":"2024-09-13T14:20:45.781344+0000","last_active":"2024-09-13T14:21:25.521997+0000","last_peered":"2024-09-13T14:21:25.521997+0000","last_clean":"2024-09-13T14:21:25.521997+0000","last_became_active":"2024-09-13T14:20:45.781053+0000","last_became_peered":"2024-09-13T14:20:45.781053+0000","last_unstale":"2024-09-13T14:21:25.521997+0000","last_undegraded":"2024-09-13T14:21:25.521997+0000","last_fullsized":"2024-09-13T14:21:25.521997+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:13.144175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520820+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:21:25.520820+0000","last_peered":"2024-09-13T14:21:25.520820+0000","last_clean":"2024-09-13T14:21:25.520820+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:21:25.520820+0000","last_undegraded":"2024-09-13T14:21:25.520820+0000","last_fullsized":"2024-09-13T14:21:25.520820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.533881+0000","last_change":"2024-09-13T14:19:32.403218+0000","last_active":"2024-09-13T14:21:26.533881+0000","last_peered":"2024-09-13T14:21:26.533881+0000","last_clean":"2024-09-13T14:21:26.533881+0000","last_became_active":"2024-09-13T14:19:27.913892+0000","last_became_peered":"2024-09-13T14:19:27.913892+0000","last_unstale":"2024-09-13T14:21:26.533881+0000","last_undegraded":"2024-09-13T14:21:26.533881+0000","last_fullsized":"2024-09-13T14:21:26.533881+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:49.778232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516005+0000","last_change":"2024-09-13T14:20:45.780258+0000","last_active":"2024-09-13T14:21:25.516005+0000","last_peered":"2024-09-13T14:21:25.516005+0000","last_clean":"2024-09-13T14:21:25.516005+0000","last_became_active":"2024-09-13T14:20:45.780013+0000","last_became_peered":"2024-09-13T14:20:45.780013+0000","last_unstale":"2024-09-13T14:21:25.516005+0000","last_undegraded":"2024-09-13T14:21:25.516005+0000","last_fullsized":"2024-09-13T14:21:25.516005+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:18.958163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516586+0000","last_change":"2024-09-13T14:20:46.788331+0000","last_active":"2024-09-13T14:21:25.516586+0000","last_peered":"2024-09-13T14:21:25.516586+0000","last_clean":"2024-09-13T14:21:25.516586+0000","last_became_active":"2024-09-13T14:20:46.788067+0000","last_became_peered":"2024-09-13T14:20:46.788067+0000","last_unstale":"2024-09-13T14:21:25.516586+0000","last_undegraded":"2024-09-13T14:21:25.516586+0000","last_fullsized":"2024-09-13T14:21:25.516586+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:07.376886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.517783+0000","last_change":"2024-09-13T14:19:32.219943+0000","last_active":"2024-09-13T14:21:26.517783+0000","last_peered":"2024-09-13T14:21:26.517783+0000","last_clean":"2024-09-13T14:21:26.517783+0000","last_became_active":"2024-09-13T14:19:28.274350+0000","last_became_peered":"2024-09-13T14:19:28.274350+0000","last_unstale":"2024-09-13T14:21:26.517783+0000","last_undegraded":"2024-09-13T14:21:26.517783+0000","last_fullsized":"2024-09-13T14:21:26.517783+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:22.369832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520069+0000","last_change":"2024-09-13T14:20:45.767155+0000","last_active":"2024-09-13T14:21:25.520069+0000","last_peered":"2024-09-13T14:21:25.520069+0000","last_clean":"2024-09-13T14:21:25.520069+0000","last_became_active":"2024-09-13T14:20:45.766745+0000","last_became_peered":"2024-09-13T14:20:45.766745+0000","last_unstale":"2024-09-13T14:21:25.520069+0000","last_undegraded":"2024-09-13T14:21:25.520069+0000","last_fullsized":"2024-09-13T14:21:25.520069+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:11.547915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520656+0000","last_change":"2024-09-13T14:20:46.806085+0000","last_active":"2024-09-13T14:21:25.520656+0000","last_peered":"2024-09-13T14:21:25.520656+0000","last_clean":"2024-09-13T14:21:25.520656+0000","last_became_active":"2024-09-13T14:20:46.805730+0000","last_became_peered":"2024-09-13T14:20:46.805730+0000","last_unstale":"2024-09-13T14:21:25.520656+0000","last_undegraded":"2024-09-13T14:21:25.520656+0000","last_fullsized":"2024-09-13T14:21:25.520656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:32.930402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.551219+0000","last_change":"2024-09-13T14:19:32.223637+0000","last_active":"2024-09-13T14:21:26.551219+0000","last_peered":"2024-09-13T14:21:26.551219+0000","last_clean":"2024-09-13T14:21:26.551219+0000","last_became_active":"2024-09-13T14:19:27.912799+0000","last_became_peered":"2024-09-13T14:19:27.912799+0000","last_unstale":"2024-09-13T14:21:26.551219+0000","last_undegraded":"2024-09-13T14:21:26.551219+0000","last_fullsized":"2024-09-13T14:21:26.551219+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:07.658550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518340+0000","last_change":"2024-09-13T14:20:45.784987+0000","last_active":"2024-09-13T14:21:25.518340+0000","last_peered":"2024-09-13T14:21:25.518340+0000","last_clean":"2024-09-13T14:21:25.518340+0000","last_became_active":"2024-09-13T14:20:45.784634+0000","last_became_peered":"2024-09-13T14:20:45.784634+0000","last_unstale":"2024-09-13T14:21:25.518340+0000","last_undegraded":"2024-09-13T14:21:25.518340+0000","last_fullsized":"2024-09-13T14:21:25.518340+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:47.241583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520376+0000","last_change":"2024-09-13T14:20:46.790997+0000","last_active":"2024-09-13T14:21:25.520376+0000","last_peered":"2024-09-13T14:21:25.520376+0000","last_clean":"2024-09-13T14:21:25.520376+0000","last_became_active":"2024-09-13T14:20:46.790767+0000","last_became_peered":"2024-09-13T14:20:46.790767+0000","last_unstale":"2024-09-13T14:21:25.520376+0000","last_undegraded":"2024-09-13T14:21:25.520376+0000","last_fullsized":"2024-09-13T14:21:25.520376+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:19:48.372643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521305+0000","last_change":"2024-09-13T14:20:45.777883+0000","last_active":"2024-09-13T14:21:25.521305+0000","last_peered":"2024-09-13T14:21:25.521305+0000","last_clean":"2024-09-13T14:21:25.521305+0000","last_became_active":"2024-09-13T14:20:45.777473+0000","last_became_peered":"2024-09-13T14:20:45.777473+0000","last_unstale":"2024-09-13T14:21:25.521305+0000","last_undegraded":"2024-09-13T14:21:25.521305+0000","last_fullsized":"2024-09-13T14:21:25.521305+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:01.376835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518338+0000","last_change":"2024-09-13T14:19:32.224472+0000","last_active":"2024-09-13T14:21:26.518338+0000","last_peered":"2024-09-13T14:21:26.518338+0000","last_clean":"2024-09-13T14:21:26.518338+0000","last_became_active":"2024-09-13T14:19:27.913130+0000","last_became_peered":"2024-09-13T14:19:27.913130+0000","last_unstale":"2024-09-13T14:21:26.518338+0000","last_undegraded":"2024-09-13T14:21:26.518338+0000","last_fullsized":"2024-09-13T14:21:26.518338+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:25.471342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520040+0000","last_change":"2024-09-13T14:20:46.805582+0000","last_active":"2024-09-13T14:21:25.520040+0000","last_peered":"2024-09-13T14:21:25.520040+0000","last_clean":"2024-09-13T14:21:25.520040+0000","last_became_active":"2024-09-13T14:20:46.805248+0000","last_became_peered":"2024-09-13T14:20:46.805248+0000","last_unstale":"2024-09-13T14:21:25.520040+0000","last_undegraded":"2024-09-13T14:21:25.520040+0000","last_fullsized":"2024-09-13T14:21:25.520040+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:58.038968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520377+0000","last_change":"2024-09-13T14:20:45.767635+0000","last_active":"2024-09-13T14:21:25.520377+0000","last_peered":"2024-09-13T14:21:25.520377+0000","last_clean":"2024-09-13T14:21:25.520377+0000","last_became_active":"2024-09-13T14:20:45.766168+0000","last_became_peered":"2024-09-13T14:20:45.766168+0000","last_unstale":"2024-09-13T14:21:25.520377+0000","last_undegraded":"2024-09-13T14:21:25.520377+0000","last_fullsized":"2024-09-13T14:21:25.520377+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:57.194458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.534312+0000","last_change":"2024-09-13T14:19:32.403220+0000","last_active":"2024-09-13T14:21:26.534312+0000","last_peered":"2024-09-13T14:21:26.534312+0000","last_clean":"2024-09-13T14:21:26.534312+0000","last_became_active":"2024-09-13T14:19:28.303245+0000","last_became_peered":"2024-09-13T14:19:28.303245+0000","last_unstale":"2024-09-13T14:21:26.534312+0000","last_undegraded":"2024-09-13T14:21:26.534312+0000","last_fullsized":"2024-09-13T14:21:26.534312+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:32.196419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516379+0000","last_change":"2024-09-13T14:20:46.805573+0000","last_active":"2024-09-13T14:21:25.516379+0000","last_peered":"2024-09-13T14:21:25.516379+0000","last_clean":"2024-09-13T14:21:25.516379+0000","last_became_active":"2024-09-13T14:20:46.805208+0000","last_became_peered":"2024-09-13T14:20:46.805208+0000","last_unstale":"2024-09-13T14:21:25.516379+0000","last_undegraded":"2024-09-13T14:21:25.516379+0000","last_fullsized":"2024-09-13T14:21:25.516379+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:10.642880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522792+0000","last_change":"2024-09-13T14:20:45.782416+0000","last_active":"2024-09-13T14:21:25.522792+0000","last_peered":"2024-09-13T14:21:25.522792+0000","last_clean":"2024-09-13T14:21:25.522792+0000","last_became_active":"2024-09-13T14:20:45.782137+0000","last_became_peered":"2024-09-13T14:20:45.782137+0000","last_unstale":"2024-09-13T14:21:25.522792+0000","last_undegraded":"2024-09-13T14:21:25.522792+0000","last_fullsized":"2024-09-13T14:21:25.522792+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:10.209630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518662+0000","last_change":"2024-09-13T14:19:32.224489+0000","last_active":"2024-09-13T14:21:26.518662+0000","last_peered":"2024-09-13T14:21:26.518662+0000","last_clean":"2024-09-13T14:21:26.518662+0000","last_became_active":"2024-09-13T14:19:28.303467+0000","last_became_peered":"2024-09-13T14:19:28.303467+0000","last_unstale":"2024-09-13T14:21:26.518662+0000","last_undegraded":"2024-09-13T14:21:26.518662+0000","last_fullsized":"2024-09-13T14:21:26.518662+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:47.015688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032689699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521762+0000","last_change":"2024-09-13T14:20:46.789886+0000","last_active":"2024-09-13T14:21:25.521762+0000","last_peered":"2024-09-13T14:21:25.521762+0000","last_clean":"2024-09-13T14:21:25.521762+0000","last_became_active":"2024-09-13T14:20:46.789537+0000","last_became_peered":"2024-09-13T14:20:46.789537+0000","last_unstale":"2024-09-13T14:21:25.521762+0000","last_undegraded":"2024-09-13T14:21:25.521762+0000","last_fullsized":"2024-09-13T14:21:25.521762+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:51.578002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516698+0000","last_change":"2024-09-13T14:20:45.788997+0000","last_active":"2024-09-13T14:21:25.516698+0000","last_peered":"2024-09-13T14:21:25.516698+0000","last_clean":"2024-09-13T14:21:25.516698+0000","last_became_active":"2024-09-13T14:20:45.787783+0000","last_became_peered":"2024-09-13T14:20:45.787783+0000","last_unstale":"2024-09-13T14:21:25.516698+0000","last_undegraded":"2024-09-13T14:21:25.516698+0000","last_fullsized":"2024-09-13T14:21:25.516698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:26.279445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518969+0000","last_change":"2024-09-13T14:19:32.224331+0000","last_active":"2024-09-13T14:21:26.518969+0000","last_peered":"2024-09-13T14:21:26.518969+0000","last_clean":"2024-09-13T14:21:26.518969+0000","last_became_active":"2024-09-13T14:19:27.914929+0000","last_became_peered":"2024-09-13T14:19:27.914929+0000","last_unstale":"2024-09-13T14:21:26.518969+0000","last_undegraded":"2024-09-13T14:21:26.518969+0000","last_fullsized":"2024-09-13T14:21:26.518969+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:47.216732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517755+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:21:25.517755+0000","last_peered":"2024-09-13T14:21:25.517755+0000","last_clean":"2024-09-13T14:21:25.517755+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:21:25.517755+0000","last_undegraded":"2024-09-13T14:21:25.517755+0000","last_fullsized":"2024-09-13T14:21:25.517755+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518114+0000","last_change":"2024-09-13T14:20:46.805671+0000","last_active":"2024-09-13T14:21:25.518114+0000","last_peered":"2024-09-13T14:21:25.518114+0000","last_clean":"2024-09-13T14:21:25.518114+0000","last_became_active":"2024-09-13T14:20:46.805292+0000","last_became_peered":"2024-09-13T14:20:46.805292+0000","last_unstale":"2024-09-13T14:21:25.518114+0000","last_undegraded":"2024-09-13T14:21:25.518114+0000","last_fullsized":"2024-09-13T14:21:25.518114+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:11.333809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518108+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:21:25.518108+0000","last_peered":"2024-09-13T14:21:25.518108+0000","last_clean":"2024-09-13T14:21:25.518108+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:21:25.518108+0000","last_undegraded":"2024-09-13T14:21:25.518108+0000","last_fullsized":"2024-09-13T14:21:25.518108+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520162+0000","last_change":"2024-09-13T14:20:46.805267+0000","last_active":"2024-09-13T14:21:25.520162+0000","last_peered":"2024-09-13T14:21:25.520162+0000","last_clean":"2024-09-13T14:21:25.520162+0000","last_became_active":"2024-09-13T14:20:46.804981+0000","last_became_peered":"2024-09-13T14:20:46.804981+0000","last_unstale":"2024-09-13T14:21:25.520162+0000","last_undegraded":"2024-09-13T14:21:25.520162+0000","last_fullsized":"2024-09-13T14:21:25.520162+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:21.247078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524310+0000","last_change":"2024-09-13T14:20:45.777690+0000","last_active":"2024-09-13T14:21:25.524310+0000","last_peered":"2024-09-13T14:21:25.524310+0000","last_clean":"2024-09-13T14:21:25.524310+0000","last_became_active":"2024-09-13T14:20:45.777421+0000","last_became_peered":"2024-09-13T14:20:45.777421+0000","last_unstale":"2024-09-13T14:21:25.524310+0000","last_undegraded":"2024-09-13T14:21:25.524310+0000","last_fullsized":"2024-09-13T14:21:25.524310+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:24.968381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521405+0000","last_change":"2024-09-13T14:20:45.772178+0000","last_active":"2024-09-13T14:21:25.521405+0000","last_peered":"2024-09-13T14:21:25.521405+0000","last_clean":"2024-09-13T14:21:25.521405+0000","last_became_active":"2024-09-13T14:20:45.771890+0000","last_became_peered":"2024-09-13T14:20:45.771890+0000","last_unstale":"2024-09-13T14:21:25.521405+0000","last_undegraded":"2024-09-13T14:21:25.521405+0000","last_fullsized":"2024-09-13T14:21:25.521405+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:43.098843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521828+0000","last_change":"2024-09-13T14:20:46.806679+0000","last_active":"2024-09-13T14:21:25.521828+0000","last_peered":"2024-09-13T14:21:25.521828+0000","last_clean":"2024-09-13T14:21:25.521828+0000","last_became_active":"2024-09-13T14:20:46.806374+0000","last_became_peered":"2024-09-13T14:20:46.806374+0000","last_unstale":"2024-09-13T14:21:25.521828+0000","last_undegraded":"2024-09-13T14:21:25.521828+0000","last_fullsized":"2024-09-13T14:21:25.521828+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:18.756579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516741+0000","last_change":"2024-09-13T14:20:45.785558+0000","last_active":"2024-09-13T14:21:25.516741+0000","last_peered":"2024-09-13T14:21:25.516741+0000","last_clean":"2024-09-13T14:21:25.516741+0000","last_became_active":"2024-09-13T14:20:45.785293+0000","last_became_peered":"2024-09-13T14:20:45.785293+0000","last_unstale":"2024-09-13T14:21:25.516741+0000","last_undegraded":"2024-09-13T14:21:25.516741+0000","last_fullsized":"2024-09-13T14:21:25.516741+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:40.291372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521739+0000","last_change":"2024-09-13T14:20:46.806862+0000","last_active":"2024-09-13T14:21:25.521739+0000","last_peered":"2024-09-13T14:21:25.521739+0000","last_clean":"2024-09-13T14:21:25.521739+0000","last_became_active":"2024-09-13T14:20:46.806563+0000","last_became_peered":"2024-09-13T14:20:46.806563+0000","last_unstale":"2024-09-13T14:21:25.521739+0000","last_undegraded":"2024-09-13T14:21:25.521739+0000","last_fullsized":"2024-09-13T14:21:25.521739+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:29.287210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520922+0000","last_change":"2024-09-13T14:20:45.788635+0000","last_active":"2024-09-13T14:21:25.520922+0000","last_peered":"2024-09-13T14:21:25.520922+0000","last_clean":"2024-09-13T14:21:25.520922+0000","last_became_active":"2024-09-13T14:20:45.788348+0000","last_became_peered":"2024-09-13T14:20:45.788348+0000","last_unstale":"2024-09-13T14:21:25.520922+0000","last_undegraded":"2024-09-13T14:21:25.520922+0000","last_fullsized":"2024-09-13T14:21:25.520922+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:05.873628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521916+0000","last_change":"2024-09-13T14:20:45.781744+0000","last_active":"2024-09-13T14:21:25.521916+0000","last_peered":"2024-09-13T14:21:25.521916+0000","last_clean":"2024-09-13T14:21:25.521916+0000","last_became_active":"2024-09-13T14:20:45.781261+0000","last_became_peered":"2024-09-13T14:20:45.781261+0000","last_unstale":"2024-09-13T14:21:25.521916+0000","last_undegraded":"2024-09-13T14:21:25.521916+0000","last_fullsized":"2024-09-13T14:21:25.521916+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:48.088892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516522+0000","last_change":"2024-09-13T14:20:46.787276+0000","last_active":"2024-09-13T14:21:25.516522+0000","last_peered":"2024-09-13T14:21:25.516522+0000","last_clean":"2024-09-13T14:21:25.516522+0000","last_became_active":"2024-09-13T14:20:46.786903+0000","last_became_peered":"2024-09-13T14:20:46.786903+0000","last_unstale":"2024-09-13T14:21:25.516522+0000","last_undegraded":"2024-09-13T14:21:25.516522+0000","last_fullsized":"2024-09-13T14:21:25.516522+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:00.187255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.519979+0000","last_change":"2024-09-13T14:20:45.773674+0000","last_active":"2024-09-13T14:21:25.519979+0000","last_peered":"2024-09-13T14:21:25.519979+0000","last_clean":"2024-09-13T14:21:25.519979+0000","last_became_active":"2024-09-13T14:20:45.773389+0000","last_became_peered":"2024-09-13T14:20:45.773389+0000","last_unstale":"2024-09-13T14:21:25.519979+0000","last_undegraded":"2024-09-13T14:21:25.519979+0000","last_fullsized":"2024-09-13T14:21:25.519979+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:38.302168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522242+0000","last_change":"2024-09-13T14:20:46.792733+0000","last_active":"2024-09-13T14:21:25.522242+0000","last_peered":"2024-09-13T14:21:25.522242+0000","last_clean":"2024-09-13T14:21:25.522242+0000","last_became_active":"2024-09-13T14:20:46.792452+0000","last_became_peered":"2024-09-13T14:20:46.792452+0000","last_unstale":"2024-09-13T14:21:25.522242+0000","last_undegraded":"2024-09-13T14:21:25.522242+0000","last_fullsized":"2024-09-13T14:21:25.522242+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:45:52.463219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517884+0000","last_change":"2024-09-13T14:20:45.774359+0000","last_active":"2024-09-13T14:21:25.517884+0000","last_peered":"2024-09-13T14:21:25.517884+0000","last_clean":"2024-09-13T14:21:25.517884+0000","last_became_active":"2024-09-13T14:20:45.774105+0000","last_became_peered":"2024-09-13T14:20:45.774105+0000","last_unstale":"2024-09-13T14:21:25.517884+0000","last_undegraded":"2024-09-13T14:21:25.517884+0000","last_fullsized":"2024-09-13T14:21:25.517884+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:51.393910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517070+0000","last_change":"2024-09-13T14:20:46.006710+0000","last_active":"2024-09-13T14:21:25.517070+0000","last_peered":"2024-09-13T14:21:25.517070+0000","last_clean":"2024-09-13T14:21:25.517070+0000","last_became_active":"2024-09-13T14:20:46.006345+0000","last_became_peered":"2024-09-13T14:20:46.006345+0000","last_unstale":"2024-09-13T14:21:25.517070+0000","last_undegraded":"2024-09-13T14:21:25.517070+0000","last_fullsized":"2024-09-13T14:21:25.517070+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:15.331017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516426+0000","last_change":"2024-09-13T14:20:45.783977+0000","last_active":"2024-09-13T14:21:25.516426+0000","last_peered":"2024-09-13T14:21:25.516426+0000","last_clean":"2024-09-13T14:21:25.516426+0000","last_became_active":"2024-09-13T14:20:45.783707+0000","last_became_peered":"2024-09-13T14:20:45.783707+0000","last_unstale":"2024-09-13T14:21:25.516426+0000","last_undegraded":"2024-09-13T14:21:25.516426+0000","last_fullsized":"2024-09-13T14:21:25.516426+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:55.084010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518033+0000","last_change":"2024-09-13T14:20:46.791058+0000","last_active":"2024-09-13T14:21:25.518033+0000","last_peered":"2024-09-13T14:21:25.518033+0000","last_clean":"2024-09-13T14:21:25.518033+0000","last_became_active":"2024-09-13T14:20:46.790804+0000","last_became_peered":"2024-09-13T14:20:46.790804+0000","last_unstale":"2024-09-13T14:21:25.518033+0000","last_undegraded":"2024-09-13T14:21:25.518033+0000","last_fullsized":"2024-09-13T14:21:25.518033+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:30.562265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520923+0000","last_change":"2024-09-13T14:20:45.773524+0000","last_active":"2024-09-13T14:21:25.520923+0000","last_peered":"2024-09-13T14:21:25.520923+0000","last_clean":"2024-09-13T14:21:25.520923+0000","last_became_active":"2024-09-13T14:20:45.773235+0000","last_became_peered":"2024-09-13T14:20:45.773235+0000","last_unstale":"2024-09-13T14:21:25.520923+0000","last_undegraded":"2024-09-13T14:21:25.520923+0000","last_fullsized":"2024-09-13T14:21:25.520923+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:57:49.544856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517851+0000","last_change":"2024-09-13T14:20:46.788891+0000","last_active":"2024-09-13T14:21:25.517851+0000","last_peered":"2024-09-13T14:21:25.517851+0000","last_clean":"2024-09-13T14:21:25.517851+0000","last_became_active":"2024-09-13T14:20:46.788681+0000","last_became_peered":"2024-09-13T14:20:46.788681+0000","last_unstale":"2024-09-13T14:21:25.517851+0000","last_undegraded":"2024-09-13T14:21:25.517851+0000","last_fullsized":"2024-09-13T14:21:25.517851+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:13.472494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521412+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:21:25.521412+0000","last_peered":"2024-09-13T14:21:25.521412+0000","last_clean":"2024-09-13T14:21:25.521412+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:21:25.521412+0000","last_undegraded":"2024-09-13T14:21:25.521412+0000","last_fullsized":"2024-09-13T14:21:25.521412+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522036+0000","last_change":"2024-09-13T14:20:46.788088+0000","last_active":"2024-09-13T14:21:25.522036+0000","last_peered":"2024-09-13T14:21:25.522036+0000","last_clean":"2024-09-13T14:21:25.522036+0000","last_became_active":"2024-09-13T14:20:46.787506+0000","last_became_peered":"2024-09-13T14:20:46.787506+0000","last_unstale":"2024-09-13T14:21:25.522036+0000","last_undegraded":"2024-09-13T14:21:25.522036+0000","last_fullsized":"2024-09-13T14:21:25.522036+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:00.167441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522181+0000","last_change":"2024-09-13T14:20:45.779325+0000","last_active":"2024-09-13T14:21:25.522181+0000","last_peered":"2024-09-13T14:21:25.522181+0000","last_clean":"2024-09-13T14:21:25.522181+0000","last_became_active":"2024-09-13T14:20:45.778792+0000","last_became_peered":"2024-09-13T14:20:45.778792+0000","last_unstale":"2024-09-13T14:21:25.522181+0000","last_undegraded":"2024-09-13T14:21:25.522181+0000","last_fullsized":"2024-09-13T14:21:25.522181+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:56.707416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.552485+0000","last_change":"2024-09-13T14:21:26.552485+0000","last_active":"2024-09-13T14:21:26.552485+0000","last_peered":"2024-09-13T14:21:26.552485+0000","last_clean":"2024-09-13T14:21:26.552485+0000","last_became_active":"2024-09-13T14:21:26.552094+0000","last_became_peered":"2024-09-13T14:21:26.552094+0000","last_unstale":"2024-09-13T14:21:26.552485+0000","last_undegraded":"2024-09-13T14:21:26.552485+0000","last_fullsized":"2024-09-13T14:21:26.552485+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:59:53.544864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516708+0000","last_change":"2024-09-13T14:20:45.787021+0000","last_active":"2024-09-13T14:21:25.516708+0000","last_peered":"2024-09-13T14:21:25.516708+0000","last_clean":"2024-09-13T14:21:25.516708+0000","last_became_active":"2024-09-13T14:20:45.786583+0000","last_became_peered":"2024-09-13T14:20:45.786583+0000","last_unstale":"2024-09-13T14:21:25.516708+0000","last_undegraded":"2024-09-13T14:21:25.516708+0000","last_fullsized":"2024-09-13T14:21:25.516708+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:45.935984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1380352,"data_stored":925744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.628},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.54500000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.50700000000000001,"15min":0.50800000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.60499999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53800000000000003,"15min":0.53400000000000003},"min":{"1min":0.443,"5min":0.374,"15min":0.374},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.47699999999999998}]},{"osd":3,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52400000000000002,"15min":0.52100000000000002},"min":{"1min":0.41599999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55500000000000005,"15min":0.55300000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.58599999999999997}]},{"osd":4,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.497},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.57399999999999995,"5min":0.622,"15min":0.622},"last":0.45600000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.438,"15min":0.436},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.60599999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.45200000000000001,"15min":0.45000000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.64600000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.66700000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495657,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1372160,"data_stored":925617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.58399999999999996},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73999999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.55000000000000004},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.52000000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66200000000000003,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70599999999999996,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.48699999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.499},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65500000000000003,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.56999999999999995}]},{"osd":4,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41299999999999998,"15min":0.40100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65400000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.317},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.30399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66600000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.27200000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42199999999999999,"15min":0.40600000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.68000000000000005,"5min":0.752,"15min":0.752},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.44,"15min":0.42199999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.68999999999999995,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.39000000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.46899999999999997,"15min":0.46800000000000003},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.40799999999999997,"5min":0.376,"15min":0.376},"max":{"1min":0.75900000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.42399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757295,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":765952,"data_stored":333936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55100000000000005,"15min":0.56799999999999995},"min":{"1min":0.376,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.379,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.745}]},{"osd":1,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.50600000000000001,"15min":0.499},"min":{"1min":0.36199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.41499999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73599999999999999,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.67100000000000004}]},{"osd":2,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.432,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68899999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.53600000000000003,"15min":0.54600000000000004},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78000000000000003,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.69999999999999996}]},{"osd":3,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68300000000000005,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76100000000000001,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.65700000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.29199999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.73799999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.40799999999999997,"15min":0.39100000000000001},"min":{"1min":0.307,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.49099999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51400000000000001,"15min":0.49399999999999999},"min":{"1min":0.36199999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.56899999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:21:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34100000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.501,"5min":0.49299999999999999,"15min":0.49099999999999999},"min":{"1min":0.32700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.76300000000000001,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.54900000000000004}]}]},{"osd":4,"up_from":30,"seq":128849018931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27788,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717364,"statfs":{"total":95995035648,"available":95966580736,"internally_reserved":0,"allocated":782336,"data_stored":335306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.40999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78200000000000003,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65100000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.34399999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77300000000000002,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.752,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.55100000000000005}]},{"osd":2,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70199999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81499999999999995,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.53800000000000003}]},{"osd":3,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84799999999999998,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.63100000000000001,"5min":0.872,"15min":0.872},"last":0.44800000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:21:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.36599999999999999},"min":{"1min":0.246,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.378},{"interface":"front","average":{"1min":0.437,"5min":0.377,"15min":0.34999999999999998},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":0.66300000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.57899999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.40100000000000002,"15min":0.38700000000000001},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48099999999999998,"15min":0.46100000000000002},"min":{"1min":0.39800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.59699999999999998}]},{"osd":7,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.626},{"interface":"front","average":{"1min":0.498,"5min":0.48599999999999999,"15min":0.48399999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.64200000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.56399999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182455,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.34799999999999998,"15min":0.35999999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.64800000000000002,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.377,"15min":0.38500000000000001},"min":{"1min":0.23000000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.249}]},{"osd":1,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.35199999999999998,"15min":0.35899999999999999},"min":{"1min":0.218,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.60499999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.35099999999999998}]},{"osd":2,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.46400000000000002,"15min":0.46800000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41299999999999998,"15min":0.41999999999999998},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.36699999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.93700000000000006,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.47499999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:21:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.57099999999999995,"15min":0.56299999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.45700000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.45500000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.0049999999999999,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73699999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.58399999999999996}]},{"osd":7,"last update":"Fri Sep 13 14:21:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.61299999999999999,"15min":0.60399999999999998},"min":{"1min":0.42299999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.60799999999999998,"15min":0.60099999999999998},"min":{"1min":0.47799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.53900000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31864,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713288,"statfs":{"total":95995035648,"available":95962406912,"internally_reserved":0,"allocated":765952,"data_stored":334212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.371,"15min":0.35099999999999998},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55500000000000005,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38700000000000001,"15min":0.36299999999999999},"min":{"1min":0.27300000000000002,"5min":0.183,"15min":0.183},"max":{"1min":0.54500000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.55100000000000005}]},{"osd":1,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.373},"min":{"1min":0.28799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":0.54900000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.26700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.51700000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.504}]},{"osd":3,"last update":"Fri Sep 13 14:21:28 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45700000000000002,"15min":0.46700000000000003},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.54800000000000004,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45900000000000002,"15min":0.47999999999999998},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55300000000000005,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63100000000000001,"15min":0.63300000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.76700000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63,"15min":0.625},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.77900000000000003}]},{"osd":5,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.80400000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60899999999999999,"15min":0.61299999999999999},"min":{"1min":0.45400000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.68700000000000006}]},{"osd":6,"last update":"Fri Sep 13 14:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88300000000000001,"5min":0.996,"15min":0.996},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.47299999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.86399999999999999,"5min":0.876,"15min":0.876},"last":0.75600000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:20:57 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.63500000000000001,"15min":0.63300000000000001},"min":{"1min":0.432,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46400000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.74299999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27756,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717396,"statfs":{"total":95995035648,"available":95966613504,"internally_reserved":0,"allocated":753664,"data_stored":333069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39400000000000002,"15min":0.35699999999999998},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58699999999999997,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.371},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.54700000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.56499999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40899999999999997,"15min":0.41599999999999998},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.54800000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.42099999999999999,"15min":0.42899999999999999},"min":{"1min":0.255,"5min":0.248,"15min":0.248},"max":{"1min":0.56299999999999994,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.45800000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.41899999999999998,"15min":0.436},"min":{"1min":0.25700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53900000000000003,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.376},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46400000000000002,"15min":0.48399999999999999},"min":{"1min":0.32300000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.61599999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.36599999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.57899999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.84999999999999998,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.45100000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.436}]},{"osd":5,"last update":"Fri Sep 13 14:21:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77000000000000002,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.41399999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.79000000000000004,"5min":0.879,"15min":0.879},"last":0.42299999999999999}]},{"osd":6,"last update":"Fri Sep 13 14:21:39 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.433,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.85099999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.71099999999999997}]},{"osd":7,"last update":"Fri Sep 13 14:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59699999999999998,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82999999999999996,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.495},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.48999999999999999,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.55000000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28348,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716804,"statfs":{"total":95995035648,"available":95966007296,"internally_reserved":0,"allocated":1359872,"data_stored":924568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:20:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.315},"min":{"1min":0.23400000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.52400000000000002,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.35699999999999998,"15min":0.33100000000000002},"min":{"1min":0.28599999999999998,"5min":0.187,"15min":0.187},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.32100000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.375,"15min":0.38300000000000001},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.68300000000000005,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40500000000000003,"15min":0.41399999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.38300000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.44400000000000001,"15min":0.45100000000000001},"min":{"1min":0.28799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66000000000000003,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.499},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.433},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.58699999999999997,"5min":0.876,"15min":0.876},"last":0.377}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.58399999999999996,"15min":0.58199999999999996},"min":{"1min":0.46999999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.54700000000000004,"15min":0.53500000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74399999999999999,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.70099999999999996}]},{"osd":6,"last update":"Fri Sep 13 14:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58899999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83199999999999996,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.44,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82499999999999996,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.52000000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.443,"5min":0.373,"15min":0.373},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.626,"15min":0.626},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.74399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:21:46.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:21:46.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:46 smithi159 ceph-mon[33330]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:47.004 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:47.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:47 smithi042 ceph-mon[29349]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:47.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:47 smithi042 ceph-mon[40707]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:47.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:47 smithi159 ceph-mon[33330]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:48.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:48 smithi042 ceph-mon[29349]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:48.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:48 smithi042 ceph-mon[40707]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:48.854 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:21:48.855 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:21:48.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:48 smithi159 ceph-mon[33330]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:49.638 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":89,"stamp":"2024-09-13T14:21:48.819800+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":228020,"kb_used_data":7756,"kb_used_omap":12,"kb_used_meta":220147,"kb_avail":749733196,"statfs":{"total":767960285184,"available":767726792704,"internally_reserved":0,"allocated":7942144,"data_stored":4446138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":225431097},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003221"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522131+0000","last_change":"2024-09-13T14:20:46.791991+0000","last_active":"2024-09-13T14:21:25.522131+0000","last_peered":"2024-09-13T14:21:25.522131+0000","last_clean":"2024-09-13T14:21:25.522131+0000","last_became_active":"2024-09-13T14:20:46.791711+0000","last_became_peered":"2024-09-13T14:20:46.791711+0000","last_unstale":"2024-09-13T14:21:25.522131+0000","last_undegraded":"2024-09-13T14:21:25.522131+0000","last_fullsized":"2024-09-13T14:21:25.522131+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:41:18.711586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524163+0000","last_change":"2024-09-13T14:20:45.776612+0000","last_active":"2024-09-13T14:21:25.524163+0000","last_peered":"2024-09-13T14:21:25.524163+0000","last_clean":"2024-09-13T14:21:25.524163+0000","last_became_active":"2024-09-13T14:20:45.776322+0000","last_became_peered":"2024-09-13T14:20:45.776322+0000","last_unstale":"2024-09-13T14:21:25.524163+0000","last_undegraded":"2024-09-13T14:21:25.524163+0000","last_fullsized":"2024-09-13T14:21:25.524163+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:02.371140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521199+0000","last_change":"2024-09-13T14:20:46.006969+0000","last_active":"2024-09-13T14:21:25.521199+0000","last_peered":"2024-09-13T14:21:25.521199+0000","last_clean":"2024-09-13T14:21:25.521199+0000","last_became_active":"2024-09-13T14:20:46.006624+0000","last_became_peered":"2024-09-13T14:20:46.006624+0000","last_unstale":"2024-09-13T14:21:25.521199+0000","last_undegraded":"2024-09-13T14:21:25.521199+0000","last_fullsized":"2024-09-13T14:21:25.521199+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:03:30.352892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516249+0000","last_change":"2024-09-13T14:20:45.790720+0000","last_active":"2024-09-13T14:21:25.516249+0000","last_peered":"2024-09-13T14:21:25.516249+0000","last_clean":"2024-09-13T14:21:25.516249+0000","last_became_active":"2024-09-13T14:20:45.790417+0000","last_became_peered":"2024-09-13T14:20:45.790417+0000","last_unstale":"2024-09-13T14:21:25.516249+0000","last_undegraded":"2024-09-13T14:21:25.516249+0000","last_fullsized":"2024-09-13T14:21:25.516249+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:57:59.219408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516773+0000","last_change":"2024-09-13T14:20:46.789778+0000","last_active":"2024-09-13T14:21:25.516773+0000","last_peered":"2024-09-13T14:21:25.516773+0000","last_clean":"2024-09-13T14:21:25.516773+0000","last_became_active":"2024-09-13T14:20:46.789470+0000","last_became_peered":"2024-09-13T14:20:46.789470+0000","last_unstale":"2024-09-13T14:21:25.516773+0000","last_undegraded":"2024-09-13T14:21:25.516773+0000","last_fullsized":"2024-09-13T14:21:25.516773+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:45.169563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524012+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:21:25.524012+0000","last_peered":"2024-09-13T14:21:25.524012+0000","last_clean":"2024-09-13T14:21:25.524012+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:21:25.524012+0000","last_undegraded":"2024-09-13T14:21:25.524012+0000","last_fullsized":"2024-09-13T14:21:25.524012+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.535073+0000","last_change":"2024-09-13T14:21:26.535073+0000","last_active":"2024-09-13T14:21:26.535073+0000","last_peered":"2024-09-13T14:21:26.535073+0000","last_clean":"2024-09-13T14:21:26.535073+0000","last_became_active":"2024-09-13T14:21:26.534784+0000","last_became_peered":"2024-09-13T14:21:26.534784+0000","last_unstale":"2024-09-13T14:21:26.535073+0000","last_undegraded":"2024-09-13T14:21:26.535073+0000","last_fullsized":"2024-09-13T14:21:26.535073+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:57.054086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521070+0000","last_change":"2024-09-13T14:20:45.777470+0000","last_active":"2024-09-13T14:21:25.521070+0000","last_peered":"2024-09-13T14:21:25.521070+0000","last_clean":"2024-09-13T14:21:25.521070+0000","last_became_active":"2024-09-13T14:20:45.777201+0000","last_became_peered":"2024-09-13T14:20:45.777201+0000","last_unstale":"2024-09-13T14:21:25.521070+0000","last_undegraded":"2024-09-13T14:21:25.521070+0000","last_fullsized":"2024-09-13T14:21:25.521070+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:16.705041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524501+0000","last_change":"2024-09-13T14:20:45.785014+0000","last_active":"2024-09-13T14:21:25.524501+0000","last_peered":"2024-09-13T14:21:25.524501+0000","last_clean":"2024-09-13T14:21:25.524501+0000","last_became_active":"2024-09-13T14:20:45.784696+0000","last_became_peered":"2024-09-13T14:20:45.784696+0000","last_unstale":"2024-09-13T14:21:25.524501+0000","last_undegraded":"2024-09-13T14:21:25.524501+0000","last_fullsized":"2024-09-13T14:21:25.524501+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:57:56.609957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520461+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:21:25.520461+0000","last_peered":"2024-09-13T14:21:25.520461+0000","last_clean":"2024-09-13T14:21:25.520461+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:21:25.520461+0000","last_undegraded":"2024-09-13T14:21:25.520461+0000","last_fullsized":"2024-09-13T14:21:25.520461+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524474+0000","last_change":"2024-09-13T14:20:45.774478+0000","last_active":"2024-09-13T14:21:25.524474+0000","last_peered":"2024-09-13T14:21:25.524474+0000","last_clean":"2024-09-13T14:21:25.524474+0000","last_became_active":"2024-09-13T14:20:45.774224+0000","last_became_peered":"2024-09-13T14:20:45.774224+0000","last_unstale":"2024-09-13T14:21:25.524474+0000","last_undegraded":"2024-09-13T14:21:25.524474+0000","last_fullsized":"2024-09-13T14:21:25.524474+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:48:07.600680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517802+0000","last_change":"2024-09-13T14:20:46.788964+0000","last_active":"2024-09-13T14:21:25.517802+0000","last_peered":"2024-09-13T14:21:25.517802+0000","last_clean":"2024-09-13T14:21:25.517802+0000","last_became_active":"2024-09-13T14:20:46.788690+0000","last_became_peered":"2024-09-13T14:20:46.788690+0000","last_unstale":"2024-09-13T14:21:25.517802+0000","last_undegraded":"2024-09-13T14:21:25.517802+0000","last_fullsized":"2024-09-13T14:21:25.517802+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:09:55.736138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524431+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:21:25.524431+0000","last_peered":"2024-09-13T14:21:25.524431+0000","last_clean":"2024-09-13T14:21:25.524431+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:21:25.524431+0000","last_undegraded":"2024-09-13T14:21:25.524431+0000","last_fullsized":"2024-09-13T14:21:25.524431+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520508+0000","last_change":"2024-09-13T14:20:46.007505+0000","last_active":"2024-09-13T14:21:25.520508+0000","last_peered":"2024-09-13T14:21:25.520508+0000","last_clean":"2024-09-13T14:21:25.520508+0000","last_became_active":"2024-09-13T14:20:46.007236+0000","last_became_peered":"2024-09-13T14:20:46.007236+0000","last_unstale":"2024-09-13T14:21:25.520508+0000","last_undegraded":"2024-09-13T14:21:25.520508+0000","last_fullsized":"2024-09-13T14:21:25.520508+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:59:49.388877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521019+0000","last_change":"2024-09-13T14:20:45.768232+0000","last_active":"2024-09-13T14:21:25.521019+0000","last_peered":"2024-09-13T14:21:25.521019+0000","last_clean":"2024-09-13T14:21:25.521019+0000","last_became_active":"2024-09-13T14:20:45.767969+0000","last_became_peered":"2024-09-13T14:20:45.767969+0000","last_unstale":"2024-09-13T14:21:25.521019+0000","last_undegraded":"2024-09-13T14:21:25.521019+0000","last_fullsized":"2024-09-13T14:21:25.521019+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:33.316679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516344+0000","last_change":"2024-09-13T14:20:46.786055+0000","last_active":"2024-09-13T14:21:25.516344+0000","last_peered":"2024-09-13T14:21:25.516344+0000","last_clean":"2024-09-13T14:21:25.516344+0000","last_became_active":"2024-09-13T14:20:46.785770+0000","last_became_peered":"2024-09-13T14:20:46.785770+0000","last_unstale":"2024-09-13T14:21:25.516344+0000","last_undegraded":"2024-09-13T14:21:25.516344+0000","last_fullsized":"2024-09-13T14:21:25.516344+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:24:00.273047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522088+0000","last_change":"2024-09-13T14:20:45.781925+0000","last_active":"2024-09-13T14:21:25.522088+0000","last_peered":"2024-09-13T14:21:25.522088+0000","last_clean":"2024-09-13T14:21:25.522088+0000","last_became_active":"2024-09-13T14:20:45.781628+0000","last_became_peered":"2024-09-13T14:20:45.781628+0000","last_unstale":"2024-09-13T14:21:25.522088+0000","last_undegraded":"2024-09-13T14:21:25.522088+0000","last_fullsized":"2024-09-13T14:21:25.522088+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:02.719731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518012+0000","last_change":"2024-09-13T14:20:46.806643+0000","last_active":"2024-09-13T14:21:25.518012+0000","last_peered":"2024-09-13T14:21:25.518012+0000","last_clean":"2024-09-13T14:21:25.518012+0000","last_became_active":"2024-09-13T14:20:46.806276+0000","last_became_peered":"2024-09-13T14:20:46.806276+0000","last_unstale":"2024-09-13T14:21:25.518012+0000","last_undegraded":"2024-09-13T14:21:25.518012+0000","last_fullsized":"2024-09-13T14:21:25.518012+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:06.830482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521036+0000","last_change":"2024-09-13T14:20:45.776337+0000","last_active":"2024-09-13T14:21:25.521036+0000","last_peered":"2024-09-13T14:21:25.521036+0000","last_clean":"2024-09-13T14:21:25.521036+0000","last_became_active":"2024-09-13T14:20:45.776009+0000","last_became_peered":"2024-09-13T14:20:45.776009+0000","last_unstale":"2024-09-13T14:21:25.521036+0000","last_undegraded":"2024-09-13T14:21:25.521036+0000","last_fullsized":"2024-09-13T14:21:25.521036+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:47.633177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518256+0000","last_change":"2024-09-13T14:20:46.791130+0000","last_active":"2024-09-13T14:21:25.518256+0000","last_peered":"2024-09-13T14:21:25.518256+0000","last_clean":"2024-09-13T14:21:25.518256+0000","last_became_active":"2024-09-13T14:20:46.790865+0000","last_became_peered":"2024-09-13T14:20:46.790865+0000","last_unstale":"2024-09-13T14:21:25.518256+0000","last_undegraded":"2024-09-13T14:21:25.518256+0000","last_fullsized":"2024-09-13T14:21:25.518256+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:36:01.653014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517986+0000","last_change":"2024-09-13T14:20:45.776700+0000","last_active":"2024-09-13T14:21:25.517986+0000","last_peered":"2024-09-13T14:21:25.517986+0000","last_clean":"2024-09-13T14:21:25.517986+0000","last_became_active":"2024-09-13T14:20:45.776221+0000","last_became_peered":"2024-09-13T14:20:45.776221+0000","last_unstale":"2024-09-13T14:21:25.517986+0000","last_undegraded":"2024-09-13T14:21:25.517986+0000","last_fullsized":"2024-09-13T14:21:25.517986+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:07.172380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.523446+0000","last_change":"2024-09-13T14:21:26.523446+0000","last_active":"2024-09-13T14:21:26.523446+0000","last_peered":"2024-09-13T14:21:26.523446+0000","last_clean":"2024-09-13T14:21:26.523446+0000","last_became_active":"2024-09-13T14:21:26.523104+0000","last_became_peered":"2024-09-13T14:21:26.523104+0000","last_unstale":"2024-09-13T14:21:26.523446+0000","last_undegraded":"2024-09-13T14:21:26.523446+0000","last_fullsized":"2024-09-13T14:21:26.523446+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:26.394962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522168+0000","last_change":"2024-09-13T14:20:45.771630+0000","last_active":"2024-09-13T14:21:25.522168+0000","last_peered":"2024-09-13T14:21:25.522168+0000","last_clean":"2024-09-13T14:21:25.522168+0000","last_became_active":"2024-09-13T14:20:45.771318+0000","last_became_peered":"2024-09-13T14:20:45.771318+0000","last_unstale":"2024-09-13T14:21:25.522168+0000","last_undegraded":"2024-09-13T14:21:25.522168+0000","last_fullsized":"2024-09-13T14:21:25.522168+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:34:14.815638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520577+0000","last_change":"2024-09-13T14:20:46.792737+0000","last_active":"2024-09-13T14:21:25.520577+0000","last_peered":"2024-09-13T14:21:25.520577+0000","last_clean":"2024-09-13T14:21:25.520577+0000","last_became_active":"2024-09-13T14:20:46.792474+0000","last_became_peered":"2024-09-13T14:20:46.792474+0000","last_unstale":"2024-09-13T14:21:25.520577+0000","last_undegraded":"2024-09-13T14:21:25.520577+0000","last_fullsized":"2024-09-13T14:21:25.520577+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:26:32.385951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524028+0000","last_change":"2024-09-13T14:18:58.652200+0000","last_active":"2024-09-13T14:21:25.524028+0000","last_peered":"2024-09-13T14:21:25.524028+0000","last_clean":"2024-09-13T14:21:25.524028+0000","last_became_active":"2024-09-13T14:18:58.651837+0000","last_became_peered":"2024-09-13T14:18:58.651837+0000","last_unstale":"2024-09-13T14:21:25.524028+0000","last_undegraded":"2024-09-13T14:21:25.524028+0000","last_fullsized":"2024-09-13T14:21:25.524028+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:10.115083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":29,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.551415+0000","last_change":"2024-09-13T14:19:32.223658+0000","last_active":"2024-09-13T14:21:26.551415+0000","last_peered":"2024-09-13T14:21:26.551415+0000","last_clean":"2024-09-13T14:21:26.551415+0000","last_became_active":"2024-09-13T14:19:27.912642+0000","last_became_peered":"2024-09-13T14:19:27.912642+0000","last_unstale":"2024-09-13T14:21:26.551415+0000","last_undegraded":"2024-09-13T14:21:26.551415+0000","last_fullsized":"2024-09-13T14:21:26.551415+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:11.887140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521997+0000","last_change":"2024-09-13T14:20:45.781344+0000","last_active":"2024-09-13T14:21:25.521997+0000","last_peered":"2024-09-13T14:21:25.521997+0000","last_clean":"2024-09-13T14:21:25.521997+0000","last_became_active":"2024-09-13T14:20:45.781053+0000","last_became_peered":"2024-09-13T14:20:45.781053+0000","last_unstale":"2024-09-13T14:21:25.521997+0000","last_undegraded":"2024-09-13T14:21:25.521997+0000","last_fullsized":"2024-09-13T14:21:25.521997+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:13.144175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520820+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:21:25.520820+0000","last_peered":"2024-09-13T14:21:25.520820+0000","last_clean":"2024-09-13T14:21:25.520820+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:21:25.520820+0000","last_undegraded":"2024-09-13T14:21:25.520820+0000","last_fullsized":"2024-09-13T14:21:25.520820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.533881+0000","last_change":"2024-09-13T14:19:32.403218+0000","last_active":"2024-09-13T14:21:26.533881+0000","last_peered":"2024-09-13T14:21:26.533881+0000","last_clean":"2024-09-13T14:21:26.533881+0000","last_became_active":"2024-09-13T14:19:27.913892+0000","last_became_peered":"2024-09-13T14:19:27.913892+0000","last_unstale":"2024-09-13T14:21:26.533881+0000","last_undegraded":"2024-09-13T14:21:26.533881+0000","last_fullsized":"2024-09-13T14:21:26.533881+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:14:49.778232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516005+0000","last_change":"2024-09-13T14:20:45.780258+0000","last_active":"2024-09-13T14:21:25.516005+0000","last_peered":"2024-09-13T14:21:25.516005+0000","last_clean":"2024-09-13T14:21:25.516005+0000","last_became_active":"2024-09-13T14:20:45.780013+0000","last_became_peered":"2024-09-13T14:20:45.780013+0000","last_unstale":"2024-09-13T14:21:25.516005+0000","last_undegraded":"2024-09-13T14:21:25.516005+0000","last_fullsized":"2024-09-13T14:21:25.516005+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:16:18.958163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516586+0000","last_change":"2024-09-13T14:20:46.788331+0000","last_active":"2024-09-13T14:21:25.516586+0000","last_peered":"2024-09-13T14:21:25.516586+0000","last_clean":"2024-09-13T14:21:25.516586+0000","last_became_active":"2024-09-13T14:20:46.788067+0000","last_became_peered":"2024-09-13T14:20:46.788067+0000","last_unstale":"2024-09-13T14:21:25.516586+0000","last_undegraded":"2024-09-13T14:21:25.516586+0000","last_fullsized":"2024-09-13T14:21:25.516586+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:07.376886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.517783+0000","last_change":"2024-09-13T14:19:32.219943+0000","last_active":"2024-09-13T14:21:26.517783+0000","last_peered":"2024-09-13T14:21:26.517783+0000","last_clean":"2024-09-13T14:21:26.517783+0000","last_became_active":"2024-09-13T14:19:28.274350+0000","last_became_peered":"2024-09-13T14:19:28.274350+0000","last_unstale":"2024-09-13T14:21:26.517783+0000","last_undegraded":"2024-09-13T14:21:26.517783+0000","last_fullsized":"2024-09-13T14:21:26.517783+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:22.369832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520069+0000","last_change":"2024-09-13T14:20:45.767155+0000","last_active":"2024-09-13T14:21:25.520069+0000","last_peered":"2024-09-13T14:21:25.520069+0000","last_clean":"2024-09-13T14:21:25.520069+0000","last_became_active":"2024-09-13T14:20:45.766745+0000","last_became_peered":"2024-09-13T14:20:45.766745+0000","last_unstale":"2024-09-13T14:21:25.520069+0000","last_undegraded":"2024-09-13T14:21:25.520069+0000","last_fullsized":"2024-09-13T14:21:25.520069+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:57:11.547915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520656+0000","last_change":"2024-09-13T14:20:46.806085+0000","last_active":"2024-09-13T14:21:25.520656+0000","last_peered":"2024-09-13T14:21:25.520656+0000","last_clean":"2024-09-13T14:21:25.520656+0000","last_became_active":"2024-09-13T14:20:46.805730+0000","last_became_peered":"2024-09-13T14:20:46.805730+0000","last_unstale":"2024-09-13T14:21:25.520656+0000","last_undegraded":"2024-09-13T14:21:25.520656+0000","last_fullsized":"2024-09-13T14:21:25.520656+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:23:32.930402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":30,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.551219+0000","last_change":"2024-09-13T14:19:32.223637+0000","last_active":"2024-09-13T14:21:26.551219+0000","last_peered":"2024-09-13T14:21:26.551219+0000","last_clean":"2024-09-13T14:21:26.551219+0000","last_became_active":"2024-09-13T14:19:27.912799+0000","last_became_peered":"2024-09-13T14:19:27.912799+0000","last_unstale":"2024-09-13T14:21:26.551219+0000","last_undegraded":"2024-09-13T14:21:26.551219+0000","last_fullsized":"2024-09-13T14:21:26.551219+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:07.658550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518340+0000","last_change":"2024-09-13T14:20:45.784987+0000","last_active":"2024-09-13T14:21:25.518340+0000","last_peered":"2024-09-13T14:21:25.518340+0000","last_clean":"2024-09-13T14:21:25.518340+0000","last_became_active":"2024-09-13T14:20:45.784634+0000","last_became_peered":"2024-09-13T14:20:45.784634+0000","last_unstale":"2024-09-13T14:21:25.518340+0000","last_undegraded":"2024-09-13T14:21:25.518340+0000","last_fullsized":"2024-09-13T14:21:25.518340+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:25:47.241583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520376+0000","last_change":"2024-09-13T14:20:46.790997+0000","last_active":"2024-09-13T14:21:25.520376+0000","last_peered":"2024-09-13T14:21:25.520376+0000","last_clean":"2024-09-13T14:21:25.520376+0000","last_became_active":"2024-09-13T14:20:46.790767+0000","last_became_peered":"2024-09-13T14:20:46.790767+0000","last_unstale":"2024-09-13T14:21:25.520376+0000","last_undegraded":"2024-09-13T14:21:25.520376+0000","last_fullsized":"2024-09-13T14:21:25.520376+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:19:48.372643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521305+0000","last_change":"2024-09-13T14:20:45.777883+0000","last_active":"2024-09-13T14:21:25.521305+0000","last_peered":"2024-09-13T14:21:25.521305+0000","last_clean":"2024-09-13T14:21:25.521305+0000","last_became_active":"2024-09-13T14:20:45.777473+0000","last_became_peered":"2024-09-13T14:20:45.777473+0000","last_unstale":"2024-09-13T14:21:25.521305+0000","last_undegraded":"2024-09-13T14:21:25.521305+0000","last_fullsized":"2024-09-13T14:21:25.521305+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:01.376835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518338+0000","last_change":"2024-09-13T14:19:32.224472+0000","last_active":"2024-09-13T14:21:26.518338+0000","last_peered":"2024-09-13T14:21:26.518338+0000","last_clean":"2024-09-13T14:21:26.518338+0000","last_became_active":"2024-09-13T14:19:27.913130+0000","last_became_peered":"2024-09-13T14:19:27.913130+0000","last_unstale":"2024-09-13T14:21:26.518338+0000","last_undegraded":"2024-09-13T14:21:26.518338+0000","last_fullsized":"2024-09-13T14:21:26.518338+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:25.471342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520040+0000","last_change":"2024-09-13T14:20:46.805582+0000","last_active":"2024-09-13T14:21:25.520040+0000","last_peered":"2024-09-13T14:21:25.520040+0000","last_clean":"2024-09-13T14:21:25.520040+0000","last_became_active":"2024-09-13T14:20:46.805248+0000","last_became_peered":"2024-09-13T14:20:46.805248+0000","last_unstale":"2024-09-13T14:21:25.520040+0000","last_undegraded":"2024-09-13T14:21:25.520040+0000","last_fullsized":"2024-09-13T14:21:25.520040+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:15:58.038968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520377+0000","last_change":"2024-09-13T14:20:45.767635+0000","last_active":"2024-09-13T14:21:25.520377+0000","last_peered":"2024-09-13T14:21:25.520377+0000","last_clean":"2024-09-13T14:21:25.520377+0000","last_became_active":"2024-09-13T14:20:45.766168+0000","last_became_peered":"2024-09-13T14:20:45.766168+0000","last_unstale":"2024-09-13T14:21:25.520377+0000","last_undegraded":"2024-09-13T14:21:25.520377+0000","last_fullsized":"2024-09-13T14:21:25.520377+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:53:57.194458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.534312+0000","last_change":"2024-09-13T14:19:32.403220+0000","last_active":"2024-09-13T14:21:26.534312+0000","last_peered":"2024-09-13T14:21:26.534312+0000","last_clean":"2024-09-13T14:21:26.534312+0000","last_became_active":"2024-09-13T14:19:28.303245+0000","last_became_peered":"2024-09-13T14:19:28.303245+0000","last_unstale":"2024-09-13T14:21:26.534312+0000","last_undegraded":"2024-09-13T14:21:26.534312+0000","last_fullsized":"2024-09-13T14:21:26.534312+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:32.196419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516379+0000","last_change":"2024-09-13T14:20:46.805573+0000","last_active":"2024-09-13T14:21:25.516379+0000","last_peered":"2024-09-13T14:21:25.516379+0000","last_clean":"2024-09-13T14:21:25.516379+0000","last_became_active":"2024-09-13T14:20:46.805208+0000","last_became_peered":"2024-09-13T14:20:46.805208+0000","last_unstale":"2024-09-13T14:21:25.516379+0000","last_undegraded":"2024-09-13T14:21:25.516379+0000","last_fullsized":"2024-09-13T14:21:25.516379+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:10.642880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522792+0000","last_change":"2024-09-13T14:20:45.782416+0000","last_active":"2024-09-13T14:21:25.522792+0000","last_peered":"2024-09-13T14:21:25.522792+0000","last_clean":"2024-09-13T14:21:25.522792+0000","last_became_active":"2024-09-13T14:20:45.782137+0000","last_became_peered":"2024-09-13T14:20:45.782137+0000","last_unstale":"2024-09-13T14:21:25.522792+0000","last_undegraded":"2024-09-13T14:21:25.522792+0000","last_fullsized":"2024-09-13T14:21:25.522792+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:34:10.209630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518662+0000","last_change":"2024-09-13T14:19:32.224489+0000","last_active":"2024-09-13T14:21:26.518662+0000","last_peered":"2024-09-13T14:21:26.518662+0000","last_clean":"2024-09-13T14:21:26.518662+0000","last_became_active":"2024-09-13T14:19:28.303467+0000","last_became_peered":"2024-09-13T14:19:28.303467+0000","last_unstale":"2024-09-13T14:21:26.518662+0000","last_undegraded":"2024-09-13T14:21:26.518662+0000","last_fullsized":"2024-09-13T14:21:26.518662+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:54:47.015688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032689699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521762+0000","last_change":"2024-09-13T14:20:46.789886+0000","last_active":"2024-09-13T14:21:25.521762+0000","last_peered":"2024-09-13T14:21:25.521762+0000","last_clean":"2024-09-13T14:21:25.521762+0000","last_became_active":"2024-09-13T14:20:46.789537+0000","last_became_peered":"2024-09-13T14:20:46.789537+0000","last_unstale":"2024-09-13T14:21:25.521762+0000","last_undegraded":"2024-09-13T14:21:25.521762+0000","last_fullsized":"2024-09-13T14:21:25.521762+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:51.578002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516698+0000","last_change":"2024-09-13T14:20:45.788997+0000","last_active":"2024-09-13T14:21:25.516698+0000","last_peered":"2024-09-13T14:21:25.516698+0000","last_clean":"2024-09-13T14:21:25.516698+0000","last_became_active":"2024-09-13T14:20:45.787783+0000","last_became_peered":"2024-09-13T14:20:45.787783+0000","last_unstale":"2024-09-13T14:21:25.516698+0000","last_undegraded":"2024-09-13T14:21:25.516698+0000","last_fullsized":"2024-09-13T14:21:25.516698+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:26.279445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.518969+0000","last_change":"2024-09-13T14:19:32.224331+0000","last_active":"2024-09-13T14:21:26.518969+0000","last_peered":"2024-09-13T14:21:26.518969+0000","last_clean":"2024-09-13T14:21:26.518969+0000","last_became_active":"2024-09-13T14:19:27.914929+0000","last_became_peered":"2024-09-13T14:19:27.914929+0000","last_unstale":"2024-09-13T14:21:26.518969+0000","last_undegraded":"2024-09-13T14:21:26.518969+0000","last_fullsized":"2024-09-13T14:21:26.518969+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:47.216732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517755+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:21:25.517755+0000","last_peered":"2024-09-13T14:21:25.517755+0000","last_clean":"2024-09-13T14:21:25.517755+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:21:25.517755+0000","last_undegraded":"2024-09-13T14:21:25.517755+0000","last_fullsized":"2024-09-13T14:21:25.517755+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518114+0000","last_change":"2024-09-13T14:20:46.805671+0000","last_active":"2024-09-13T14:21:25.518114+0000","last_peered":"2024-09-13T14:21:25.518114+0000","last_clean":"2024-09-13T14:21:25.518114+0000","last_became_active":"2024-09-13T14:20:46.805292+0000","last_became_peered":"2024-09-13T14:20:46.805292+0000","last_unstale":"2024-09-13T14:21:25.518114+0000","last_undegraded":"2024-09-13T14:21:25.518114+0000","last_fullsized":"2024-09-13T14:21:25.518114+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:11.333809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"71'6","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518108+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:21:25.518108+0000","last_peered":"2024-09-13T14:21:25.518108+0000","last_clean":"2024-09-13T14:21:25.518108+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:21:25.518108+0000","last_undegraded":"2024-09-13T14:21:25.518108+0000","last_fullsized":"2024-09-13T14:21:25.518108+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520162+0000","last_change":"2024-09-13T14:20:46.805267+0000","last_active":"2024-09-13T14:21:25.520162+0000","last_peered":"2024-09-13T14:21:25.520162+0000","last_clean":"2024-09-13T14:21:25.520162+0000","last_became_active":"2024-09-13T14:20:46.804981+0000","last_became_peered":"2024-09-13T14:20:46.804981+0000","last_unstale":"2024-09-13T14:21:25.520162+0000","last_undegraded":"2024-09-13T14:21:25.520162+0000","last_fullsized":"2024-09-13T14:21:25.520162+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:25:21.247078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.524310+0000","last_change":"2024-09-13T14:20:45.777690+0000","last_active":"2024-09-13T14:21:25.524310+0000","last_peered":"2024-09-13T14:21:25.524310+0000","last_clean":"2024-09-13T14:21:25.524310+0000","last_became_active":"2024-09-13T14:20:45.777421+0000","last_became_peered":"2024-09-13T14:20:45.777421+0000","last_unstale":"2024-09-13T14:21:25.524310+0000","last_undegraded":"2024-09-13T14:21:25.524310+0000","last_fullsized":"2024-09-13T14:21:25.524310+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:24.968381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521405+0000","last_change":"2024-09-13T14:20:45.772178+0000","last_active":"2024-09-13T14:21:25.521405+0000","last_peered":"2024-09-13T14:21:25.521405+0000","last_clean":"2024-09-13T14:21:25.521405+0000","last_became_active":"2024-09-13T14:20:45.771890+0000","last_became_peered":"2024-09-13T14:20:45.771890+0000","last_unstale":"2024-09-13T14:21:25.521405+0000","last_undegraded":"2024-09-13T14:21:25.521405+0000","last_fullsized":"2024-09-13T14:21:25.521405+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:43.098843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521828+0000","last_change":"2024-09-13T14:20:46.806679+0000","last_active":"2024-09-13T14:21:25.521828+0000","last_peered":"2024-09-13T14:21:25.521828+0000","last_clean":"2024-09-13T14:21:25.521828+0000","last_became_active":"2024-09-13T14:20:46.806374+0000","last_became_peered":"2024-09-13T14:20:46.806374+0000","last_unstale":"2024-09-13T14:21:25.521828+0000","last_undegraded":"2024-09-13T14:21:25.521828+0000","last_fullsized":"2024-09-13T14:21:25.521828+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:18.756579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516741+0000","last_change":"2024-09-13T14:20:45.785558+0000","last_active":"2024-09-13T14:21:25.516741+0000","last_peered":"2024-09-13T14:21:25.516741+0000","last_clean":"2024-09-13T14:21:25.516741+0000","last_became_active":"2024-09-13T14:20:45.785293+0000","last_became_peered":"2024-09-13T14:20:45.785293+0000","last_unstale":"2024-09-13T14:21:25.516741+0000","last_undegraded":"2024-09-13T14:21:25.516741+0000","last_fullsized":"2024-09-13T14:21:25.516741+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:27:40.291372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521739+0000","last_change":"2024-09-13T14:20:46.806862+0000","last_active":"2024-09-13T14:21:25.521739+0000","last_peered":"2024-09-13T14:21:25.521739+0000","last_clean":"2024-09-13T14:21:25.521739+0000","last_became_active":"2024-09-13T14:20:46.806563+0000","last_became_peered":"2024-09-13T14:20:46.806563+0000","last_unstale":"2024-09-13T14:21:25.521739+0000","last_undegraded":"2024-09-13T14:21:25.521739+0000","last_fullsized":"2024-09-13T14:21:25.521739+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:20:29.287210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520922+0000","last_change":"2024-09-13T14:20:45.788635+0000","last_active":"2024-09-13T14:21:25.520922+0000","last_peered":"2024-09-13T14:21:25.520922+0000","last_clean":"2024-09-13T14:21:25.520922+0000","last_became_active":"2024-09-13T14:20:45.788348+0000","last_became_peered":"2024-09-13T14:20:45.788348+0000","last_unstale":"2024-09-13T14:21:25.520922+0000","last_undegraded":"2024-09-13T14:21:25.520922+0000","last_fullsized":"2024-09-13T14:21:25.520922+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:05.873628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521916+0000","last_change":"2024-09-13T14:20:45.781744+0000","last_active":"2024-09-13T14:21:25.521916+0000","last_peered":"2024-09-13T14:21:25.521916+0000","last_clean":"2024-09-13T14:21:25.521916+0000","last_became_active":"2024-09-13T14:20:45.781261+0000","last_became_peered":"2024-09-13T14:20:45.781261+0000","last_unstale":"2024-09-13T14:21:25.521916+0000","last_undegraded":"2024-09-13T14:21:25.521916+0000","last_fullsized":"2024-09-13T14:21:25.521916+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:48.088892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516522+0000","last_change":"2024-09-13T14:20:46.787276+0000","last_active":"2024-09-13T14:21:25.516522+0000","last_peered":"2024-09-13T14:21:25.516522+0000","last_clean":"2024-09-13T14:21:25.516522+0000","last_became_active":"2024-09-13T14:20:46.786903+0000","last_became_peered":"2024-09-13T14:20:46.786903+0000","last_unstale":"2024-09-13T14:21:25.516522+0000","last_undegraded":"2024-09-13T14:21:25.516522+0000","last_fullsized":"2024-09-13T14:21:25.516522+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:00.187255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.519979+0000","last_change":"2024-09-13T14:20:45.773674+0000","last_active":"2024-09-13T14:21:25.519979+0000","last_peered":"2024-09-13T14:21:25.519979+0000","last_clean":"2024-09-13T14:21:25.519979+0000","last_became_active":"2024-09-13T14:20:45.773389+0000","last_became_peered":"2024-09-13T14:20:45.773389+0000","last_unstale":"2024-09-13T14:21:25.519979+0000","last_undegraded":"2024-09-13T14:21:25.519979+0000","last_fullsized":"2024-09-13T14:21:25.519979+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:38.302168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522242+0000","last_change":"2024-09-13T14:20:46.792733+0000","last_active":"2024-09-13T14:21:25.522242+0000","last_peered":"2024-09-13T14:21:25.522242+0000","last_clean":"2024-09-13T14:21:25.522242+0000","last_became_active":"2024-09-13T14:20:46.792452+0000","last_became_peered":"2024-09-13T14:20:46.792452+0000","last_unstale":"2024-09-13T14:21:25.522242+0000","last_undegraded":"2024-09-13T14:21:25.522242+0000","last_fullsized":"2024-09-13T14:21:25.522242+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:45:52.463219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517884+0000","last_change":"2024-09-13T14:20:45.774359+0000","last_active":"2024-09-13T14:21:25.517884+0000","last_peered":"2024-09-13T14:21:25.517884+0000","last_clean":"2024-09-13T14:21:25.517884+0000","last_became_active":"2024-09-13T14:20:45.774105+0000","last_became_peered":"2024-09-13T14:20:45.774105+0000","last_unstale":"2024-09-13T14:21:25.517884+0000","last_undegraded":"2024-09-13T14:21:25.517884+0000","last_fullsized":"2024-09-13T14:21:25.517884+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:51.393910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517070+0000","last_change":"2024-09-13T14:20:46.006710+0000","last_active":"2024-09-13T14:21:25.517070+0000","last_peered":"2024-09-13T14:21:25.517070+0000","last_clean":"2024-09-13T14:21:25.517070+0000","last_became_active":"2024-09-13T14:20:46.006345+0000","last_became_peered":"2024-09-13T14:20:46.006345+0000","last_unstale":"2024-09-13T14:21:25.517070+0000","last_undegraded":"2024-09-13T14:21:25.517070+0000","last_fullsized":"2024-09-13T14:21:25.517070+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:15:15.331017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516426+0000","last_change":"2024-09-13T14:20:45.783977+0000","last_active":"2024-09-13T14:21:25.516426+0000","last_peered":"2024-09-13T14:21:25.516426+0000","last_clean":"2024-09-13T14:21:25.516426+0000","last_became_active":"2024-09-13T14:20:45.783707+0000","last_became_peered":"2024-09-13T14:20:45.783707+0000","last_unstale":"2024-09-13T14:21:25.516426+0000","last_undegraded":"2024-09-13T14:21:25.516426+0000","last_fullsized":"2024-09-13T14:21:25.516426+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:55.084010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.518033+0000","last_change":"2024-09-13T14:20:46.791058+0000","last_active":"2024-09-13T14:21:25.518033+0000","last_peered":"2024-09-13T14:21:25.518033+0000","last_clean":"2024-09-13T14:21:25.518033+0000","last_became_active":"2024-09-13T14:20:46.790804+0000","last_became_peered":"2024-09-13T14:20:46.790804+0000","last_unstale":"2024-09-13T14:21:25.518033+0000","last_undegraded":"2024-09-13T14:21:25.518033+0000","last_fullsized":"2024-09-13T14:21:25.518033+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:30.562265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.520923+0000","last_change":"2024-09-13T14:20:45.773524+0000","last_active":"2024-09-13T14:21:25.520923+0000","last_peered":"2024-09-13T14:21:25.520923+0000","last_clean":"2024-09-13T14:21:25.520923+0000","last_became_active":"2024-09-13T14:20:45.773235+0000","last_became_peered":"2024-09-13T14:20:45.773235+0000","last_unstale":"2024-09-13T14:21:25.520923+0000","last_undegraded":"2024-09-13T14:21:25.520923+0000","last_fullsized":"2024-09-13T14:21:25.520923+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:57:49.544856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.517851+0000","last_change":"2024-09-13T14:20:46.788891+0000","last_active":"2024-09-13T14:21:25.517851+0000","last_peered":"2024-09-13T14:21:25.517851+0000","last_clean":"2024-09-13T14:21:25.517851+0000","last_became_active":"2024-09-13T14:20:46.788681+0000","last_became_peered":"2024-09-13T14:20:46.788681+0000","last_unstale":"2024-09-13T14:21:25.517851+0000","last_undegraded":"2024-09-13T14:21:25.517851+0000","last_fullsized":"2024-09-13T14:21:25.517851+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:13.472494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":17,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.521412+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:21:25.521412+0000","last_peered":"2024-09-13T14:21:25.521412+0000","last_clean":"2024-09-13T14:21:25.521412+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:21:25.521412+0000","last_undegraded":"2024-09-13T14:21:25.521412+0000","last_fullsized":"2024-09-13T14:21:25.521412+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522036+0000","last_change":"2024-09-13T14:20:46.788088+0000","last_active":"2024-09-13T14:21:25.522036+0000","last_peered":"2024-09-13T14:21:25.522036+0000","last_clean":"2024-09-13T14:21:25.522036+0000","last_became_active":"2024-09-13T14:20:46.787506+0000","last_became_peered":"2024-09-13T14:20:46.787506+0000","last_unstale":"2024-09-13T14:21:25.522036+0000","last_undegraded":"2024-09-13T14:21:25.522036+0000","last_fullsized":"2024-09-13T14:21:25.522036+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:00.167441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":16,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.522181+0000","last_change":"2024-09-13T14:20:45.779325+0000","last_active":"2024-09-13T14:21:25.522181+0000","last_peered":"2024-09-13T14:21:25.522181+0000","last_clean":"2024-09-13T14:21:25.522181+0000","last_became_active":"2024-09-13T14:20:45.778792+0000","last_became_peered":"2024-09-13T14:20:45.778792+0000","last_unstale":"2024-09-13T14:21:25.522181+0000","last_undegraded":"2024-09-13T14:21:25.522181+0000","last_fullsized":"2024-09-13T14:21:25.522181+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:55:56.707416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-13T14:21:26.552485+0000","last_change":"2024-09-13T14:21:26.552485+0000","last_active":"2024-09-13T14:21:26.552485+0000","last_peered":"2024-09-13T14:21:26.552485+0000","last_clean":"2024-09-13T14:21:26.552485+0000","last_became_active":"2024-09-13T14:21:26.552094+0000","last_became_peered":"2024-09-13T14:21:26.552094+0000","last_unstale":"2024-09-13T14:21:26.552485+0000","last_undegraded":"2024-09-13T14:21:26.552485+0000","last_fullsized":"2024-09-13T14:21:26.552485+0000","mapping_epoch":72,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":73,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:59:53.544864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-09-13T14:21:25.516708+0000","last_change":"2024-09-13T14:20:45.787021+0000","last_active":"2024-09-13T14:21:25.516708+0000","last_peered":"2024-09-13T14:21:25.516708+0000","last_clean":"2024-09-13T14:21:25.516708+0000","last_became_active":"2024-09-13T14:20:45.786583+0000","last_became_peered":"2024-09-13T14:20:45.786583+0000","last_unstale":"2024-09-13T14:21:25.516708+0000","last_undegraded":"2024-09-13T14:21:25.516708+0000","last_fullsized":"2024-09-13T14:21:25.516708+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:41:45.935984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234020,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1380352,"data_stored":925744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53300000000000003,"15min":0.53200000000000003},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.44900000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.50700000000000001,"15min":0.50800000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.52100000000000002}]},{"osd":2,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.53800000000000003,"15min":0.53400000000000003},"min":{"1min":0.443,"5min":0.374,"15min":0.374},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.503},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.49299999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52400000000000002,"15min":0.52100000000000002},"min":{"1min":0.41599999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55500000000000005,"15min":0.55300000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53500000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60299999999999998,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.57399999999999995,"5min":0.622,"15min":0.622},"last":0.59399999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.438,"15min":0.436},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.60599999999999998,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.45200000000000001,"15min":0.45000000000000001},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.47499999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.46200000000000002,"15min":0.45900000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.49399999999999999,"15min":0.49299999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.63300000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495658,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1340,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1372160,"data_stored":925617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.56899999999999995,"15min":0.58399999999999996},"min":{"1min":0.36799999999999999,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73999999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"last":0.48999999999999999}]},{"osd":1,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.55000000000000004},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.119,"5min":1.119,"15min":1.119},"last":0.373},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52300000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.56499999999999995}]},{"osd":2,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.51900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66200000000000003,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.36499999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70599999999999996,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.433}]},{"osd":3,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.1419999999999999,"5min":1.1419999999999999,"15min":1.1419999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.54200000000000004,"15min":0.55000000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65500000000000003,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.52700000000000002}]},{"osd":4,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41299999999999998,"15min":0.40100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65400000000000003,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.40999999999999998,"15min":0.39800000000000002},"min":{"1min":0.30399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.66600000000000004,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50700000000000001}]},{"osd":5,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42199999999999999,"15min":0.40600000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.68000000000000005,"5min":0.752,"15min":0.752},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.44,"15min":0.42199999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.68999999999999995,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.55000000000000004}]},{"osd":7,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.46899999999999997,"15min":0.46800000000000003},"min":{"1min":0.33700000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.68100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.54200000000000004,"15min":0.53900000000000003},"min":{"1min":0.40799999999999997,"5min":0.376,"15min":0.376},"max":{"1min":0.75900000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57699999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757296,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":765952,"data_stored":333936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.55100000000000005,"15min":0.56799999999999995},"min":{"1min":0.376,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.67500000000000004,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.52500000000000002,"15min":0.51900000000000002},"min":{"1min":0.379,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58999999999999997}]},{"osd":1,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.50600000000000001,"15min":0.499},"min":{"1min":0.36199999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.41499999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.73599999999999999,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.627}]},{"osd":2,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.432,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68899999999999995,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.53600000000000003,"15min":0.54600000000000004},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78000000000000003,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.52200000000000002}]},{"osd":3,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68300000000000005,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.76100000000000001,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.61299999999999999}]},{"osd":4,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.40400000000000003,"15min":0.39400000000000002},"min":{"1min":0.29199999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.73799999999999999,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.311},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.40799999999999997,"15min":0.39100000000000001},"min":{"1min":0.307,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.32400000000000001}]},{"osd":6,"last update":"Fri Sep 13 14:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.45200000000000001,"15min":0.436},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.51400000000000001,"15min":0.49399999999999999},"min":{"1min":0.36199999999999999,"5min":0.316,"15min":0.316},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.39100000000000001}]},{"osd":7,"last update":"Fri Sep 13 14:21:00 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34100000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.49299999999999999,"15min":0.49099999999999999},"min":{"1min":0.32700000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.76300000000000001,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.374}]}]},{"osd":4,"up_from":30,"seq":128849018932,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27788,"kb_used_data":764,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717364,"statfs":{"total":95995035648,"available":95966580736,"internally_reserved":0,"allocated":782336,"data_stored":335306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.40999999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78200000000000003,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.65100000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.52400000000000002}]},{"osd":1,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.77300000000000002,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.53300000000000003,"15min":0.53400000000000003},"min":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.752,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.63200000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.35099999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.70199999999999996,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56000000000000005,"15min":0.56200000000000006},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81499999999999995,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.62}]},{"osd":3,"last update":"Fri Sep 13 14:20:53 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.435,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84799999999999998,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.53000000000000003},"min":{"1min":0.37,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.63100000000000001,"5min":0.872,"15min":0.872},"last":0.55900000000000005}]},{"osd":5,"last update":"Fri Sep 13 14:21:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.38200000000000001,"15min":0.36599999999999999},"min":{"1min":0.246,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.627,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.437,"5min":0.377,"15min":0.34999999999999998},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":0.66300000000000003,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.443}]},{"osd":6,"last update":"Fri Sep 13 14:21:35 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.40100000000000002,"15min":0.38700000000000001},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.496},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.48099999999999998,"15min":0.46100000000000002},"min":{"1min":0.39800000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.57199999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:20:58 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51300000000000001,"15min":0.51600000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.498,"5min":0.48599999999999999,"15min":0.48399999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.64200000000000002,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.60399999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182456,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":333686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.33000000000000002,"5min":0.34799999999999998,"15min":0.35999999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.64800000000000002,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.377,"15min":0.38500000000000001},"min":{"1min":0.23000000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.32700000000000001}]},{"osd":1,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.35199999999999998,"15min":0.35899999999999999},"min":{"1min":0.218,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.433},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.60499999999999998,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.377}]},{"osd":2,"last update":"Fri Sep 13 14:21:33 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.46400000000000002,"15min":0.46800000000000003},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.41299999999999998,"15min":0.41999999999999998},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34300000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.53600000000000003,"15min":0.52700000000000002},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.93700000000000006,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.5},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.59499999999999997}]},{"osd":5,"last update":"Fri Sep 13 14:21:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.57099999999999995,"15min":0.56299999999999994},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.60299999999999998}]},{"osd":6,"last update":"Fri Sep 13 14:21:37 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.60499999999999998,"15min":0.60299999999999998},"min":{"1min":0.45500000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.0049999999999999,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58599999999999997,"15min":0.57499999999999996},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.73699999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.58199999999999996}]},{"osd":7,"last update":"Fri Sep 13 14:21:01 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.61299999999999999,"15min":0.60399999999999998},"min":{"1min":0.42299999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.60799999999999998,"15min":0.60099999999999998},"min":{"1min":0.47799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.101,"5min":1.101,"15min":1.101},"last":0.57299999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31864,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713288,"statfs":{"total":95995035648,"available":95962406912,"internally_reserved":0,"allocated":765952,"data_stored":334212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.371,"15min":0.35099999999999998},"min":{"1min":0.28399999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.55500000000000005,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38700000000000001,"15min":0.36299999999999999},"min":{"1min":0.27300000000000002,"5min":0.183,"15min":0.183},"max":{"1min":0.54500000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.41499999999999998}]},{"osd":1,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.39900000000000002,"15min":0.373},"min":{"1min":0.28799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":0.54900000000000004,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.26700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.51700000000000002,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38700000000000001}]},{"osd":3,"last update":"Fri Sep 13 14:21:28 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.45700000000000002,"15min":0.46700000000000003},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.54800000000000004,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.433},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.45900000000000002,"15min":0.47999999999999998},"min":{"1min":0.318,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.55300000000000005,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.374}]},{"osd":4,"last update":"Fri Sep 13 14:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.63100000000000001,"15min":0.63300000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.63,"15min":0.625},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.65100000000000002}]},{"osd":5,"last update":"Fri Sep 13 14:21:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.60899999999999999,"15min":0.61299999999999999},"min":{"1min":0.45400000000000001,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.52500000000000002}]},{"osd":6,"last update":"Fri Sep 13 14:21:38 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.63600000000000001,"15min":0.63400000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88300000000000001,"5min":0.996,"15min":0.996},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.65400000000000003,"15min":0.65300000000000002},"min":{"1min":0.47299999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.86399999999999999,"5min":0.876,"15min":0.876},"last":0.68899999999999995}]},{"osd":7,"last update":"Fri Sep 13 14:20:57 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.63500000000000001,"15min":0.63300000000000001},"min":{"1min":0.432,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"min":{"1min":0.46400000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.51600000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27756,"kb_used_data":736,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717396,"statfs":{"total":95995035648,"available":95966613504,"internally_reserved":0,"allocated":753664,"data_stored":333069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 14:20:47 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.39400000000000002,"15min":0.35699999999999998},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.58699999999999997,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.38500000000000001,"15min":0.371},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.54700000000000004,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.22700000000000001}]},{"osd":2,"last update":"Fri Sep 13 14:21:06 2024","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40899999999999997,"15min":0.41599999999999998},"min":{"1min":0.27100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.54800000000000004,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.42099999999999999,"15min":0.42899999999999999},"min":{"1min":0.255,"5min":0.248,"15min":0.248},"max":{"1min":0.56299999999999994,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.29899999999999999}]},{"osd":3,"last update":"Fri Sep 13 14:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.41899999999999998,"15min":0.436},"min":{"1min":0.25700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.53900000000000003,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.46400000000000002,"15min":0.48399999999999999},"min":{"1min":0.32300000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.61599999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.32100000000000001}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.57899999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.84999999999999998,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.45100000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.60399999999999998}]},{"osd":5,"last update":"Fri Sep 13 14:21:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.56699999999999995,"15min":0.56599999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.77000000000000002,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57899999999999996,"15min":0.57499999999999996},"min":{"1min":0.41399999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.79000000000000004,"5min":0.879,"15min":0.879},"last":0.59199999999999997}]},{"osd":6,"last update":"Fri Sep 13 14:21:39 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.433,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.85099999999999998,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.65000000000000002}]},{"osd":7,"last update":"Fri Sep 13 14:20:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59699999999999998,"15min":0.58999999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82999999999999996,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.48999999999999999,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.63100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28348,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716804,"statfs":{"total":95995035648,"available":95966007296,"internally_reserved":0,"allocated":1359872,"data_stored":924568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 14:21:45 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"min":{"1min":0.25,"5min":0.22,"15min":0.22},"max":{"1min":0.58299999999999996,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.373,"5min":0.36699999999999999,"15min":0.33500000000000002},"min":{"1min":0.19900000000000001,"5min":0.187,"15min":0.187},"max":{"1min":0.52000000000000002,"5min":0.627,"15min":0.627},"last":0.495}]},{"osd":2,"last update":"Fri Sep 13 14:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.375,"15min":0.38300000000000001},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.68300000000000005,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40500000000000003,"15min":0.41399999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.622}]},{"osd":3,"last update":"Fri Sep 13 14:21:32 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.44400000000000001,"15min":0.45100000000000001},"min":{"1min":0.28799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66000000000000003,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42899999999999999,"15min":0.433},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.58699999999999997,"5min":0.876,"15min":0.876},"last":0.66100000000000003}]},{"osd":4,"last update":"Fri Sep 13 14:20:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.58399999999999996,"15min":0.58199999999999996},"min":{"1min":0.46999999999999997,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.57899999999999996,"15min":0.57299999999999995},"min":{"1min":0.40500000000000003,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Fri Sep 13 14:21:16 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.54700000000000004,"15min":0.53500000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.56499999999999995,"15min":0.56399999999999995},"min":{"1min":0.42599999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.74399999999999999,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.68000000000000005}]},{"osd":6,"last update":"Fri Sep 13 14:21:36 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58899999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.83199999999999996,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.44,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.82499999999999996,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.73999999999999999}]},{"osd":7,"last update":"Fri Sep 13 14:21:03 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.57099999999999995,"15min":0.56399999999999995},"min":{"1min":0.443,"5min":0.373,"15min":0.373},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.626,"15min":0.626},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.72099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:21:49.643 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:21:49.643 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:21:49.643 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:21:49.643 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:21:49.859 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:50.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:50 smithi042 ceph-mon[29349]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:50.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:50 smithi042 ceph-mon[29349]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:50 smithi042 ceph-mon[40707]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:50.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:50 smithi042 ceph-mon[40707]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:50.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:50 smithi159 ceph-mon[33330]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:50 smithi159 ceph-mon[33330]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:21:51.702 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:21:51.703 INFO:teuthology.orchestra.run.smithi042.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:21:52.450 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:21:52.451 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:21:52.460 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-13T14:21:52.460 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:21:52.469 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:21:52.469 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:21:52.470 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-13T14:21:52.471 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:21:52.472 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:21:52.473 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-13T14:21:52.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-13T14:21:52.473 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:21:52.473 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T14:21:52.474 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T14:21:52.474 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-13T14:21:52.705 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:52.712 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:52.740 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:52 smithi042 ceph-mon[40707]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:52.740 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:52 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3807343408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:21:52.740 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:52 smithi042 ceph-mon[29349]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:52.740 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:52 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3807343408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:21:52.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:52 smithi159 ceph-mon[33330]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:52.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:52 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3807343408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T14:21:53.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:21:53] "GET /metrics HTTP/1.1" 200 33713 "" "Prometheus/2.43.0" 2024-09-13T14:21:54.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:54 smithi042 ceph-mon[29349]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:54.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:54 smithi042 ceph-mon[40707]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:54.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:54 smithi159 ceph-mon[33330]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:55.181 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE Bus STOPPING 2024-09-13T14:21:55.181 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:21:55.181 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE Bus STOPPED 2024-09-13T14:21:55.181 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE Bus STARTING 2024-09-13T14:21:55.182 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE Serving on http://:::9283 2024-09-13T14:21:55.182 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:21:55] ENGINE Bus STARTED 2024-09-13T14:21:55.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:55.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1593330752' entity='client.admin' 2024-09-13T14:21:55.799 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:55.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:55.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1593330752' entity='client.admin' 2024-09-13T14:21:55.800 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:21:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:55 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1593330752' entity='client.admin' 2024-09-13T14:21:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:21:56.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T14:21:56.652 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:56.773 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[40707]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:56.773 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:56.773 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:56.774 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:56.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[29349]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:56.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:56.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:56.774 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:56.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:56 smithi159 ceph-mon[33330]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:56 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:21:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:56 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:21:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:21:57.596 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_0' created 2024-09-13T14:21:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:57 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3823408198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:57 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:57 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3823408198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:57 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:57.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:57 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3823408198' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:57 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-09-13T14:21:58.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[29349]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:58.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:21:58.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[29349]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:21:58.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[40707]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:58.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:21:58.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:58 smithi042 ceph-mon[40707]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:21:58.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:58 smithi159 ceph-mon[33330]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:21:58.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:58 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-13T14:21:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:58 smithi159 ceph-mon[33330]: osdmap e74: 8 total, 8 up, 8 in 2024-09-13T14:21:59.158 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-13T14:21:59.463 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:21:59.825 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:21:59 smithi042 ceph-mon[40707]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:21:59.825 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:21:59 smithi042 ceph-mon[29349]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:21:59.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-09-13T14:21:59.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:21:59 smithi159 ceph-mon[33330]: osdmap e75: 8 total, 8 up, 8 in 2024-09-13T14:22:00.600 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[29349]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[29349]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[40707]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[40707]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:22:00.724 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:00 smithi042 ceph-mon[40707]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:22:00.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:00 smithi159 ceph-mon[33330]: pgmap v96: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:00 smithi159 ceph-mon[33330]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:22:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:00 smithi159 ceph-mon[33330]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T14:22:02.719 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:02 smithi042 ceph-mon[29349]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:02.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:02 smithi042 ceph-mon[40707]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:02.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:02 smithi159 ceph-mon[33330]: pgmap v98: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:03.231 INFO:teuthology.orchestra.run.smithi042.stdout:wait for mgr daemons to upgrade 2024-09-13T14:22:03.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:03] "GET /metrics HTTP/1.1" 200 33712 "" "Prometheus/2.43.0" 2024-09-13T14:22:03.616 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-13T14:22:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:03 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:22:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:03 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:22:03.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-13T14:22:03.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:03 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:22:04.684 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[29349]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[29349]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[40707]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:22:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:04 smithi042 ceph-mon[40707]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:22:04.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:04 smithi159 ceph-mon[33330]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:04 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/664565671' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:22:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:04 smithi159 ceph-mon[33330]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T14:22:05.580 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-13T14:22:05.583 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:22:05.583 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:22:05.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:22:05.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:22:05.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_85522 2024-09-13T14:22:05.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:22:05.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:22:05.801 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:05 smithi042 ceph-mon[40707]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:05.802 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:05 smithi042 ceph-mon[40707]: Cluster is now healthy 2024-09-13T14:22:05.802 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:05 smithi042 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:05.802 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:05 smithi042 ceph-mon[29349]: Cluster is now healthy 2024-09-13T14:22:05.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:05 smithi159 ceph-mon[33330]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:22:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:05 smithi159 ceph-mon[33330]: Cluster is now healthy 2024-09-13T14:22:06.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 4152 4136 258.485 258.5 0.00385018 0.00385564 2024-09-13T14:22:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:06 smithi042 ceph-mon[40707]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:06 smithi042 ceph-mon[29349]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:06 smithi159 ceph-mon[33330]: pgmap v101: 89 pgs: 89 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-13T14:22:07.658 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 7907 7891 246.575 234.688 0.00232621 0.00404553 2024-09-13T14:22:08.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 11397 11381 237.077 218.125 0.00512629 0.00421053 2024-09-13T14:22:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:08 smithi159 ceph-mon[33330]: pgmap v102: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2024-09-13T14:22:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:08 smithi159 ceph-mon[33330]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:08 smithi042 ceph-mon[29349]: pgmap v102: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2024-09-13T14:22:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:08 smithi042 ceph-mon[29349]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:08 smithi042 ceph-mon[40707]: pgmap v102: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s wr, 80 op/s 2024-09-13T14:22:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:08 smithi042 ceph-mon[40707]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:09.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 14639 14623 228.46 202.625 0.00680343 0.00437049 2024-09-13T14:22:10.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 17674 17658 220.702 189.688 0.00228301 0.00452461 2024-09-13T14:22:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:10 smithi159 ceph-mon[33330]: pgmap v103: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2024-09-13T14:22:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:10 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:10 smithi042 ceph-mon[29349]: pgmap v103: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2024-09-13T14:22:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:10 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:10 smithi042 ceph-mon[40707]: pgmap v103: 89 pgs: 89 active+clean; 41 MiB data, 223 MiB used, 715 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2024-09-13T14:22:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:10 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:11.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 20329 20313 211.572 165.938 0.00480427 0.00472011 2024-09-13T14:22:12.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 22726 22710 202.748 149.812 0.00576582 0.00492586 2024-09-13T14:22:13.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:12 smithi159 ceph-mon[33330]: pgmap v104: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:13.079 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:12 smithi042 ceph-mon[29349]: pgmap v104: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:13.079 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:12 smithi042 ceph-mon[40707]: pgmap v104: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:13.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:13] "GET /metrics HTTP/1.1" 200 36054 "" "Prometheus/2.43.0" 2024-09-13T14:22:13.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 15 25155 25140 196.387 151.875 0.00573643 0.00508616 2024-09-13T14:22:14.659 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 27623 27607 191.696 154.188 0.00266453 0.00519224 2024-09-13T14:22:14.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:14 smithi159 ceph-mon[33330]: pgmap v105: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:14 smithi042 ceph-mon[40707]: pgmap v105: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:14 smithi042 ceph-mon[29349]: pgmap v105: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:22:15.660 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 30028 30012 187.541 150.312 0.00676484 0.00532586 2024-09-13T14:22:16.660 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 32397 32381 183.951 148.062 0.00727067 0.0054316 2024-09-13T14:22:16.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:16 smithi159 ceph-mon[33330]: pgmap v106: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:22:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:16 smithi042 ceph-mon[29349]: pgmap v106: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:22:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:16 smithi042 ceph-mon[40707]: pgmap v106: 89 pgs: 89 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2024-09-13T14:22:17.660 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 34690 34674 180.563 143.312 0.0119871 0.00553368 2024-09-13T14:22:18.660 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 36901 36885 177.303 138.188 0.0037035 0.00563511 2024-09-13T14:22:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:18 smithi159 ceph-mon[33330]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-13T14:22:19.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:18 smithi042 ceph-mon[29349]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-13T14:22:19.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:18 smithi042 ceph-mon[40707]: pgmap v107: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-13T14:22:19.661 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 15 39006 38991 174.035 131.625 0.00913428 0.00574052 2024-09-13T14:22:20.661 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 41130 41114 171.278 132.688 0.00550564 0.00583391 2024-09-13T14:22:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:20 smithi159 ceph-mon[33330]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-13T14:22:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:20 smithi042 ceph-mon[40707]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-13T14:22:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:20 smithi042 ceph-mon[29349]: pgmap v108: 89 pgs: 89 active+clean; 1.9 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s 2024-09-13T14:22:21.661 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 43210 43194 168.697 130 0.00787062 0.0059231 2024-09-13T14:22:22.662 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 45229 45213 166.187 126.188 0.00546529 0.00601229 2024-09-13T14:22:22.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:22 smithi159 ceph-mon[33330]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 212 MiB/s wr, 3.40k op/s 2024-09-13T14:22:23.078 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:22 smithi042 ceph-mon[40707]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 212 MiB/s wr, 3.40k op/s 2024-09-13T14:22:23.097 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:22 smithi042 ceph-mon[29349]: pgmap v109: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 212 MiB/s wr, 3.40k op/s 2024-09-13T14:22:23.350 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:23] "GET /metrics HTTP/1.1" 200 36054 "" "Prometheus/2.43.0" 2024-09-13T14:22:23.663 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 47101 47085 163.445 117 0.00986525 0.00611305 2024-09-13T14:22:24.663 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 49035 49019 161.202 120.875 0.00312952 0.00619915 2024-09-13T14:22:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:24 smithi159 ceph-mon[33330]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:24 smithi042 ceph-mon[40707]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:24 smithi042 ceph-mon[29349]: pgmap v110: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:25.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:22:25.664052+0000 min lat: 0.00123693 max lat: 0.0597992 avg lat: 0.00628293 2024-09-13T14:22:25.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:22:25.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 15 50921 50906 159.038 117.938 0.00959004 0.00628293 2024-09-13T14:22:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:26.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 52745 52729 156.89 113.938 0.00998587 0.006369 2024-09-13T14:22:26.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:26 smithi159 ceph-mon[33330]: pgmap v111: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:26 smithi042 ceph-mon[40707]: pgmap v111: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:26 smithi042 ceph-mon[29349]: pgmap v111: 89 pgs: 89 active+clean; 2.5 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-09-13T14:22:27.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 54565 54549 154.929 113.75 0.00887042 0.00644939 2024-09-13T14:22:28.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 56278 56262 152.848 107.062 0.00570278 0.00653764 2024-09-13T14:22:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:28 smithi159 ceph-mon[33330]: pgmap v112: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-13T14:22:29.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:28 smithi042 ceph-mon[29349]: pgmap v112: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-13T14:22:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:28 smithi042 ceph-mon[40707]: pgmap v112: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 172 MiB/s wr, 2.76k op/s 2024-09-13T14:22:29.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 57951 57935 150.836 104.562 0.0147351 0.00662398 2024-09-13T14:22:30.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 59579 59563 148.872 101.75 0.00226649 0.00670372 2024-09-13T14:22:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:30 smithi159 ceph-mon[33330]: pgmap v113: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:22:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:30 smithi042 ceph-mon[40707]: pgmap v113: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:22:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:30 smithi042 ceph-mon[29349]: pgmap v113: 89 pgs: 89 active+clean; 3.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:22:31.664 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 61096 61080 146.792 94.8125 0.0103201 0.00680775 2024-09-13T14:22:32.668 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 62637 62621 144.918 96.3125 0.0112569 0.00689505 2024-09-13T14:22:32.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:32 smithi159 ceph-mon[33330]: pgmap v114: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:22:33.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:32 smithi042 ceph-mon[40707]: pgmap v114: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:22:33.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:32 smithi042 ceph-mon[29349]: pgmap v114: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:22:33.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:33] "GET /metrics HTTP/1.1" 200 36065 "" "Prometheus/2.43.0" 2024-09-13T14:22:33.666 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 64158 64142 143.132 95.0625 0.0171548 0.0069816 2024-09-13T14:22:34.667 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 65647 65631 141.401 93.0625 0.00966042 0.00706707 2024-09-13T14:22:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:34 smithi042 ceph-mon[29349]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:34 smithi042 ceph-mon[40707]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:35.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:34 smithi159 ceph-mon[33330]: pgmap v115: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:35.670 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 67135 67119 139.779 93 0.00970072 0.00714918 2024-09-13T14:22:36.672 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 15 68544 68529 138.111 88.125 0.0125842 0.00723621 2024-09-13T14:22:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:36 smithi042 ceph-mon[29349]: pgmap v116: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:36 smithi042 ceph-mon[40707]: pgmap v116: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:36 smithi159 ceph-mon[33330]: pgmap v116: 89 pgs: 89 active+clean; 3.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:37.670 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 69966 69950 136.571 88.8125 0.0100818 0.00731769 2024-09-13T14:22:38.671 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 71325 71309 135.002 84.9375 0.006687 0.00740195 2024-09-13T14:22:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:38 smithi042 ceph-mon[40707]: pgmap v117: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-13T14:22:39.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:38 smithi042 ceph-mon[29349]: pgmap v117: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-13T14:22:39.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:38 smithi159 ceph-mon[33330]: pgmap v117: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2024-09-13T14:22:39.671 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 72706 72690 133.57 86.3125 0.0152103 0.00748177 2024-09-13T14:22:40.671 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 74072 74056 132.192 85.375 0.0101914 0.00755953 2024-09-13T14:22:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:40 smithi042 ceph-mon[29349]: pgmap v118: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:22:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:40 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:41.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:40 smithi042 ceph-mon[40707]: pgmap v118: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:22:41.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:40 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:40 smithi159 ceph-mon[33330]: pgmap v118: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:22:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:40 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:41.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 75434 75418 130.883 85.125 0.00751814 0.00763501 2024-09-13T14:22:42.672 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 76778 76762 129.616 84 0.0276776 0.00770969 2024-09-13T14:22:43.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:42 smithi042 ceph-mon[40707]: pgmap v119: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:22:43.077 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:42 smithi042 ceph-mon[29349]: pgmap v119: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:22:43.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:42 smithi159 ceph-mon[33330]: pgmap v119: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:22:43.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:43] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-09-13T14:22:43.672 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 78260 78244 128.643 92.625 0.00926525 0.00776901 2024-09-13T14:22:44.672 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 79699 79683 127.65 89.9375 0.0131286 0.00782891 2024-09-13T14:22:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:44 smithi042 ceph-mon[40707]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:22:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:44 smithi042 ceph-mon[29349]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:22:45.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:44 smithi159 ceph-mon[33330]: pgmap v120: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.43k op/s 2024-09-13T14:22:45.380 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:45.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:22:45.673057+0000 min lat: 0.00123693 max lat: 0.189223 avg lat: 0.00789005 2024-09-13T14:22:45.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:22:45.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 81108 81092 126.661 88.0625 0.015597 0.00789005 2024-09-13T14:22:46.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:22:46.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:46 smithi159 ceph-mon[33330]: pgmap v121: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:22:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:46 smithi159 ceph-mon[33330]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:46 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:46 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:46.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 82406 82390 125.55 81.125 0.00388206 0.00796014 2024-09-13T14:22:46.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[40707]: pgmap v121: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:22:46.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[40707]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:46.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:46.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:46.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[29349]: pgmap v121: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:22:46.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[29349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:46.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:46.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:46 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:47.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 83652 83636 124.415 77.875 0.00837962 0.00803332 2024-09-13T14:22:48.137 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:22:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:48 smithi042 ceph-mon[40707]: pgmap v122: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-13T14:22:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:48 smithi042 ceph-mon[29349]: pgmap v122: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-13T14:22:48.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 15 84848 84833 123.261 74.8125 0.0176063 0.00810835 2024-09-13T14:22:48.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:48 smithi159 ceph-mon[33330]: pgmap v122: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-09-13T14:22:49.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 86045 86029 122.159 74.75 0.00497345 0.0081808 2024-09-13T14:22:50.558 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:50 smithi042 ceph-mon[40707]: pgmap v123: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:22:50.558 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:50 smithi042 ceph-mon[29349]: pgmap v123: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:22:50.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 87347 87331 121.253 81.375 0.0135782 0.00824248 2024-09-13T14:22:50.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:50 smithi159 ceph-mon[33330]: pgmap v123: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:22:51.127 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:22:51.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 88591 88575 120.307 77.75 0.0118972 0.00830689 2024-09-13T14:22:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:51 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:51.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:51 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:51.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:51 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:51.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:51 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:51 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:51 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (2m) 94s ago 3m 16.9M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (2m) 1s ago 2m 94.2M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (119s) 94s ago 118s 16.4M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (116s) 1s ago 115s 15.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,8765 running (7m) 1s ago 7m 439M - 18.2.4-843-g2c01458a c129f5a2fd4e 6d297500e0d0 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283,8765 running (8m) 94s ago 8m 483M - 18.2.4-843-g2c01458a c129f5a2fd4e f7442a44ace3 2024-09-13T14:22:52.158 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (8m) 94s ago 8m 43.8M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (7m) 1s ago 7m 40.0M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (7m) 94s ago 7m 40.8M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (2m) 94s ago 2m 15.5M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (2m) 1s ago 2m 15.7M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (6m) 94s ago 6m 43.4M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (6m) 94s ago 6m 43.6M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (6m) 94s ago 5m 50.7M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (5m) 94s ago 5m 41.5M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (5m) 1s ago 5m 237M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (4m) 1s ago 4m 264M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (4m) 1s ago 4m 141M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (4m) 1s ago 4m 169M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:22:52.159 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (2m) 1s ago 3m 45.3M - 2.43.0 a07b618ecd1d 96d161cf02d4 2024-09-13T14:22:52.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 89825 89809 119.388 77.125 0.01169 0.00837084 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:22:52.792 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 15 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:22:52.793 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:22:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[29349]: pgmap v124: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[29349]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[29349]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[40707]: pgmap v124: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[40707]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:52 smithi042 ceph-mon[40707]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:52 smithi159 ceph-mon[33330]: pgmap v124: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:22:52.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:52 smithi159 ceph-mon[33330]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:52.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:52 smithi159 ceph-mon[33330]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:53.323 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "", 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:22:53.324 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:22:53.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:22:53] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-09-13T14:22:53.673 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 91046 91030 118.491 76.3125 0.0137733 0.0084343 2024-09-13T14:22:53.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/86797731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:53.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/86797731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:53.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:53.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:53 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/86797731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:22:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:53 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:54.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 92349 92333 117.735 81.4375 0.00503933 0.0084885 2024-09-13T14:22:54.806 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[40707]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:22:54.806 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[40707]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:54.806 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[40707]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:54.807 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[29349]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:22:54.807 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[29349]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:54.807 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:54 smithi042 ceph-mon[29349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:54.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:54 smithi159 ceph-mon[33330]: pgmap v125: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s 2024-09-13T14:22:54.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:54 smithi159 ceph-mon[33330]: from='client.14850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:22:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:54 smithi159 ceph-mon[33330]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:22:55.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 15 93675 93660 117.039 82.9375 0.00871032 0.00853901 2024-09-13T14:22:55.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:55.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:22:56.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 94805 94789 116.128 70.5625 0.0136958 0.00860607 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[40707]: pgmap v126: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[29349]: pgmap v126: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:56.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:56 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:56 smithi159 ceph-mon[33330]: pgmap v126: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:22:57.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:57.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:56 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:22:57.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:56 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:22:57.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 95988 95972 115.316 73.9375 0.00883971 0.00866675 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[40707]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[40707]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[40707]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[40707]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi159 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[29349]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[29349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:22:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:22:57.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:57 smithi042 ceph-mon[29349]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi159 2024-09-13T14:22:57.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:57 smithi159 ceph-mon[33330]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (unknown) 2024-09-13T14:22:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:57 smithi159 ceph-mon[33330]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:22:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:57 smithi159 ceph-mon[33330]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T14:22:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:57 smithi159 ceph-mon[33330]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f on smithi159 2024-09-13T14:22:58.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 97087 97071 114.437 68.6875 0.00679108 0.00873311 2024-09-13T14:22:58.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:22:58 smithi042 ceph-mon[40707]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:22:58.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:22:58 smithi042 ceph-mon[29349]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:22:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:22:58 smithi159 ceph-mon[33330]: pgmap v127: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-13T14:22:59.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 98181 98165 113.584 68.375 0.0114112 0.00879923 2024-09-13T14:23:00.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 99274 99258 112.761 68.3125 0.00772905 0.00886355 2024-09-13T14:23:00.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:00 smithi042 ceph-mon[40707]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:23:00.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:00 smithi042 ceph-mon[29349]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:23:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:00 smithi159 ceph-mon[33330]: pgmap v128: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:23:01.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 100367 100351 111.967 68.3125 0.010649 0.0089265 2024-09-13T14:23:02.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 101419 101403 111.156 65.75 0.00758302 0.00899135 2024-09-13T14:23:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:02 smithi042 ceph-mon[29349]: pgmap v129: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:23:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:02 smithi042 ceph-mon[40707]: pgmap v129: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:23:02.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:02 smithi159 ceph-mon[33330]: pgmap v129: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:23:03.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:03] "GET /metrics HTTP/1.1" 200 36071 "" "Prometheus/2.43.0" 2024-09-13T14:23:03.674 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 102467 102451 110.369 65.5 0.0279358 0.00905548 2024-09-13T14:23:04.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 103379 103363 109.465 57 0.0114403 0.00913031 2024-09-13T14:23:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:04 smithi042 ceph-mon[29349]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:04 smithi042 ceph-mon[40707]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:04 smithi159 ceph-mon[33330]: pgmap v130: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:05.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:23:05.674938+0000 min lat: 0.00123693 max lat: 0.189223 avg lat: 0.00920337 2024-09-13T14:23:05.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:23:05.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 104280 104264 108.579 56.3125 0.00869876 0.00920337 2024-09-13T14:23:06.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 105302 105286 107.846 63.875 0.0212743 0.00926757 2024-09-13T14:23:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:06 smithi042 ceph-mon[40707]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T14:23:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:06 smithi042 ceph-mon[29349]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T14:23:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:06 smithi159 ceph-mon[33330]: pgmap v131: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2024-09-13T14:23:07.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 106309 106293 107.122 62.9375 0.0241725 0.00933041 2024-09-13T14:23:08.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 107331 107315 106.435 63.875 0.00933111 0.00938964 2024-09-13T14:23:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:08 smithi159 ceph-mon[33330]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:23:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:08 smithi042 ceph-mon[29349]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:23:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:08 smithi042 ceph-mon[40707]: pgmap v132: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:23:09.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 108361 108345 105.778 64.375 0.0141972 0.00944833 2024-09-13T14:23:10.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 15 109350 109335 105.103 61.875 0.0293639 0.0095097 2024-09-13T14:23:10.899 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:10 smithi042 ceph-mon[40707]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:23:10.899 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:10 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:10.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:10 smithi042 ceph-mon[29349]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:23:10.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:10 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:10 smithi159 ceph-mon[33330]: pgmap v133: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s 2024-09-13T14:23:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:10 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:11.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 110372 110356 104.477 63.8125 0.0171785 0.00956623 2024-09-13T14:23:12.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 111376 111360 103.854 62.75 0.0189621 0.00962357 2024-09-13T14:23:12.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:12 smithi159 ceph-mon[33330]: pgmap v134: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:23:13.077 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:12 smithi042 ceph-mon[29349]: pgmap v134: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:23:13.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:12 smithi042 ceph-mon[40707]: pgmap v134: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-13T14:23:13.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:13] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.43.0" 2024-09-13T14:23:13.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 112389 112373 103.258 63.3125 0.00630543 0.00967892 2024-09-13T14:23:14.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 113419 113403 102.695 64.375 0.0278826 0.00973261 2024-09-13T14:23:14.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:14 smithi159 ceph-mon[33330]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T14:23:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:14 smithi042 ceph-mon[40707]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T14:23:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:14 smithi042 ceph-mon[29349]: pgmap v135: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-09-13T14:23:15.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 15 114400 114385 102.104 61.375 0.017143 0.00978863 2024-09-13T14:23:16.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 115386 115370 101.533 61.5625 0.0108021 0.00984343 2024-09-13T14:23:16.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:16 smithi159 ceph-mon[33330]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:23:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:16 smithi042 ceph-mon[40707]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:23:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:16 smithi042 ceph-mon[29349]: pgmap v136: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s 2024-09-13T14:23:17.675 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 116401 116385 101.004 63.4375 0.00501082 0.00989576 2024-09-13T14:23:18.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 117371 117355 100.451 60.625 0.0050491 0.00994979 2024-09-13T14:23:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:18 smithi159 ceph-mon[33330]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:23:19.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:18 smithi042 ceph-mon[40707]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:23:19.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:18 smithi042 ceph-mon[29349]: pgmap v137: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:23:19.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 118317 118301 99.8928 59.125 0.00902593 0.0100056 2024-09-13T14:23:20.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 119290 119274 99.3717 60.8125 0.0459586 0.0100573 2024-09-13T14:23:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:20 smithi159 ceph-mon[33330]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2024-09-13T14:23:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:20 smithi042 ceph-mon[40707]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2024-09-13T14:23:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:20 smithi042 ceph-mon[29349]: pgmap v138: 89 pgs: 89 active+clean; 7.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s 2024-09-13T14:23:21.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 120182 120166 98.7977 55.75 0.0145177 0.0101166 2024-09-13T14:23:22.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 121082 121066 98.2452 56.25 0.0127179 0.0101732 2024-09-13T14:23:22.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:22 smithi159 ceph-mon[33330]: pgmap v139: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:23:23.076 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:22 smithi042 ceph-mon[40707]: pgmap v139: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:23:23.077 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:22 smithi042 ceph-mon[29349]: pgmap v139: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:23:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:23] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.43.0" 2024-09-13T14:23:23.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 121962 121946 97.6908 55 0.0186553 0.0102317 2024-09-13T14:23:23.787 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:23:24.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 122821 122805 97.1338 53.6875 0.0527749 0.0102898 2024-09-13T14:23:24.698 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:23:24.698 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (2m) 31s ago 3m 27.7M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:23:24.698 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (3m) 34s ago 3m 94.2M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (2m) 31s ago 2m 17.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (2m) 34s ago 2m 15.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,8765 running (7m) 34s ago 7m 439M - 18.2.4-843-g2c01458a c129f5a2fd4e 6d297500e0d0 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283,8765 running (9m) 31s ago 9m 489M - 18.2.4-843-g2c01458a c129f5a2fd4e f7442a44ace3 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (9m) 31s ago 9m 46.6M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (8m) 34s ago 7m 40.0M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (7m) 31s ago 7m 41.4M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (3m) 31s ago 3m 15.7M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (3m) 34s ago 3m 15.7M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (7m) 31s ago 7m 240M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:23:24.699 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (6m) 31s ago 6m 166M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (6m) 31s ago 6m 212M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (6m) 31s ago 6m 118M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (5m) 34s ago 5m 237M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (5m) 34s ago 5m 264M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (4m) 34s ago 4m 141M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (4m) 34s ago 4m 169M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:23:24.700 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (2m) 34s ago 4m 45.3M - 2.43.0 a07b618ecd1d 96d161cf02d4 2024-09-13T14:23:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:24 smithi159 ceph-mon[33330]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:24 smithi159 ceph-mon[33330]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:24 smithi042 ceph-mon[40707]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:24 smithi042 ceph-mon[40707]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:24 smithi042 ceph-mon[29349]: pgmap v140: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:23:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:24 smithi042 ceph-mon[29349]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:23:25.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 15 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:23:25.264 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:23:25.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:23:25.676668+0000 min lat: 0.00123693 max lat: 0.189223 avg lat: 0.0103458 2024-09-13T14:23:25.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:23:25.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 123700 123684 96.6064 54.9375 0.0218657 0.0103458 2024-09-13T14:23:25.727 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:23:25.727 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:23:25.727 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:23:25.727 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:23:25.727 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:23:25.728 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "0/2 daemons upgraded", 2024-09-13T14:23:25.728 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f image on host smithi159", 2024-09-13T14:23:25.728 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:23:25.728 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:23:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:25 smithi159 ceph-mon[33330]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:25 smithi159 ceph-mon[33330]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:25 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:25 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2804584221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[40707]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[40707]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2804584221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[29349]: from='client.24637 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[29349]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:26.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:25 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2804584221' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:26.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 124549 124533 96.0688 53.0625 0.00635401 0.0104037 2024-09-13T14:23:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:26 smithi159 ceph-mon[33330]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:23:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:26 smithi159 ceph-mon[33330]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:26 smithi042 ceph-mon[40707]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:23:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:26 smithi042 ceph-mon[40707]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:26 smithi042 ceph-mon[29349]: pgmap v141: 89 pgs: 89 active+clean; 7.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:23:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:26 smithi042 ceph-mon[29349]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:27.676 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 125439 125423 95.5756 55.625 0.00719748 0.0104583 2024-09-13T14:23:28.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 126314 126298 95.0829 54.6875 0.019536 0.010512 2024-09-13T14:23:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:28 smithi159 ceph-mon[33330]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:23:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:28 smithi042 ceph-mon[40707]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:23:29.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:28 smithi042 ceph-mon[29349]: pgmap v142: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:23:29.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 127170 127154 94.5879 53.5 0.0197902 0.0105657 2024-09-13T14:23:30.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 128038 128022 94.1134 54.25 0.038912 0.0106197 2024-09-13T14:23:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:30 smithi159 ceph-mon[33330]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2024-09-13T14:23:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:30 smithi042 ceph-mon[40707]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2024-09-13T14:23:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:30 smithi042 ceph-mon[29349]: pgmap v143: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 772 op/s 2024-09-13T14:23:31.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 128865 128849 93.6201 51.6875 0.0164097 0.0106765 2024-09-13T14:23:32.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 129831 129815 93.2379 60.375 0.0249229 0.0107207 2024-09-13T14:23:32.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:32 smithi159 ceph-mon[33330]: pgmap v144: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2024-09-13T14:23:33.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:32 smithi042 ceph-mon[40707]: pgmap v144: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2024-09-13T14:23:33.077 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:32 smithi042 ceph-mon[29349]: pgmap v144: 89 pgs: 89 active+clean; 7.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 50 MiB/s wr, 793 op/s 2024-09-13T14:23:33.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:33] "GET /metrics HTTP/1.1" 200 36073 "" "Prometheus/2.43.0" 2024-09-13T14:23:33.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 130715 130699 92.8062 55.25 0.00366654 0.0107632 2024-09-13T14:23:34.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 131517 131501 92.3266 50.125 0.00941285 0.010826 2024-09-13T14:23:34.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:34 smithi159 ceph-mon[33330]: pgmap v145: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:23:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:34 smithi042 ceph-mon[40707]: pgmap v145: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:23:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:34 smithi042 ceph-mon[29349]: pgmap v145: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:23:35.677 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0067 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 132512 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 92.0154 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 43.2441 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 258.5 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 50.125 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1472 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 691.906 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 4136 2024-09-13T14:23:35.678 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 802 2024-09-13T14:23:35.679 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0108637 2024-09-13T14:23:35.679 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.00766663 2024-09-13T14:23:35.679 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 0.255951 2024-09-13T14:23:35.679 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00123693 2024-09-13T14:23:35.682 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:23:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:36 smithi159 ceph-mon[33330]: pgmap v146: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s 2024-09-13T14:23:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:36 smithi042 ceph-mon[40707]: pgmap v146: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s 2024-09-13T14:23:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:36 smithi042 ceph-mon[29349]: pgmap v146: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 68 MiB/s wr, 1.10k op/s 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: Upgrade: Updating mgr.x 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:38 smithi159 ceph-mon[33330]: Deploying daemon mgr.x on smithi159 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 68 MiB/s wr, 1.10k op/s 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: Upgrade: Updating mgr.x 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[40707]: Deploying daemon mgr.x on smithi159 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: pgmap v147: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 68 MiB/s wr, 1.10k op/s 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: Upgrade: Updating mgr.x 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:23:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:38 smithi042 ceph-mon[29349]: Deploying daemon mgr.x on smithi159 2024-09-13T14:23:39.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:39 smithi159 systemd[1]: Stopping Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:23:40.302 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:40 smithi159 ceph-mon[33330]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 737 op/s 2024-09-13T14:23:40.303 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:40 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:40.303 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 podman[69407]: 2024-09-13 14:23:40.004361564 +0000 UTC m=+0.532905212 container died 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T14:23:40.404 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:40 smithi042 ceph-mon[40707]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 737 op/s 2024-09-13T14:23:40.404 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:40 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:40.404 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:40 smithi042 ceph-mon[29349]: pgmap v148: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 737 op/s 2024-09-13T14:23:40.404 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:40 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:40.719 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 podman[69407]: 2024-09-13 14:23:40.416471068 +0000 UTC m=+0.945014706 container cleanup 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:23:40.720 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 bash[69407]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:23:40.971 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 podman[69422]: 2024-09-13 14:23:40.718772818 +0000 UTC m=+0.708996660 container remove 6d297500e0d0bae3b56afa6151aab9220d32bc6ae2661ec043642a331b77c445 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:23:40.971 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:23:41.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:23:41.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 systemd[1]: Stopped Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:23:41.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:40 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Consumed 11.212s CPU time. 2024-09-13T14:23:41.638 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:41 smithi159 systemd[1]: Starting Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:23:41.983 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:41 smithi159 podman[69541]: 2024-09-13 14:23:41.544040683 +0000 UTC m=+0.037270994 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:23:41.983 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:41 smithi159 podman[69541]: 2024-09-13 14:23:41.686575626 +0000 UTC m=+0.179805921 container create 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T14:23:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:42 smithi159 ceph-mon[33330]: pgmap v149: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 764 op/s 2024-09-13T14:23:42.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:41 smithi159 podman[69541]: 2024-09-13 14:23:41.982164703 +0000 UTC m=+0.475395013 container init 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T14:23:42.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:41 smithi159 podman[69541]: 2024-09-13 14:23:41.995936602 +0000 UTC m=+0.489166907 container start 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:23:42.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:42 smithi159 bash[69541]: 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 2024-09-13T14:23:42.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:42 smithi159 systemd[1]: Started Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:23:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:42 smithi042 ceph-mon[40707]: pgmap v149: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 764 op/s 2024-09-13T14:23:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:42 smithi042 ceph-mon[29349]: pgmap v149: 89 pgs: 89 active+clean; 8.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 764 op/s 2024-09-13T14:23:43.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:43] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-09-13T14:23:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:43 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:43 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:43 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:43 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:23:44.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:44 smithi159 ceph-mon[33330]: pgmap v150: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.13k op/s 2024-09-13T14:23:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:44 smithi042 ceph-mon[40707]: pgmap v150: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.13k op/s 2024-09-13T14:23:44.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:44 smithi042 ceph-mon[29349]: pgmap v150: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 1.13k op/s 2024-09-13T14:23:46.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:46 smithi159 ceph-mon[33330]: pgmap v151: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 791 op/s 2024-09-13T14:23:46.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:46 smithi042 ceph-mon[29349]: pgmap v151: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 791 op/s 2024-09-13T14:23:46.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:46 smithi042 ceph-mon[40707]: pgmap v151: 89 pgs: 89 active+clean; 7.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 791 op/s 2024-09-13T14:23:48.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:48 smithi159 ceph-mon[33330]: pgmap v152: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.15k op/s 2024-09-13T14:23:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:48 smithi042 ceph-mon[40707]: pgmap v152: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.15k op/s 2024-09-13T14:23:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:48 smithi042 ceph-mon[29349]: pgmap v152: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 22 MiB/s wr, 1.15k op/s 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:49 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:49 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:49 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:49 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:49.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:49 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:23:50.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:50 smithi159 ceph-mon[33330]: pgmap v153: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T14:23:50.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:50 smithi042 ceph-mon[29349]: pgmap v153: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T14:23:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:50 smithi042 ceph-mon[40707]: pgmap v153: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 771 op/s 2024-09-13T14:23:51.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:23:50] ENGINE Bus STARTING 2024-09-13T14:23:51.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: CherryPy Checker: 2024-09-13T14:23:51.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: The Application mounted at '' has an empty config. 2024-09-13T14:23:51.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:23:51.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:23:51] ENGINE Serving on http://:::9283 2024-09-13T14:23:51.250 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:23:51] ENGINE Bus STARTED 2024-09-13T14:23:51.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: Standby manager daemon x restarted 2024-09-13T14:23:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:23:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:51 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: Standby manager daemon x restarted 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: Standby manager daemon x restarted 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: Standby manager daemon x started 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:23:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:51 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:23:52.667 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:52 smithi159 ceph-mon[33330]: pgmap v154: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:23:52.667 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:52 smithi159 ceph-mon[33330]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-13T14:23:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:52 smithi042 ceph-mon[29349]: pgmap v154: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:23:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:52 smithi042 ceph-mon[29349]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-13T14:23:52.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:52 smithi042 ceph-mon[40707]: pgmap v154: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2024-09-13T14:23:52.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:52 smithi042 ceph-mon[40707]: mgrmap e22: y(active, since 4m), standbys: x 2024-09-13T14:23:53.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:23:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:23:53] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-09-13T14:23:54.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:54 smithi159 ceph-mon[33330]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:23:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:54 smithi042 ceph-mon[29349]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:23:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:54 smithi042 ceph-mon[40707]: pgmap v155: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:23:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:55 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:55 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:55 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:23:56.212 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:23:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:56 smithi159 ceph-mon[33330]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:23:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:56 smithi042 ceph-mon[40707]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:23:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:56 smithi042 ceph-mon[29349]: pgmap v156: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 697 op/s 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (3m) 63s ago 4m 27.7M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (4m) 8s ago 4m 94.2M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (3m) 63s ago 3m 17.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (3m) 8s ago 3m 16.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (15s) 8s ago 8m 59.7M - 19.3.0-4697-gd58f7306 fec07016a2b0 222e56781225 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283,8765 running (9m) 63s ago 9m 489M - 18.2.4-843-g2c01458a c129f5a2fd4e f7442a44ace3 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (9m) 63s ago 10m 46.6M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (8m) 8s ago 8m 38.9M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:23:57.168 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (8m) 63s ago 8m 41.4M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (3m) 63s ago 3m 15.7M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (3m) 8s ago 3m 15.7M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (7m) 63s ago 7m 240M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (7m) 63s ago 7m 166M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (7m) 63s ago 7m 212M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (6m) 63s ago 6m 118M 4017M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (6m) 8s ago 6m 354M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (5m) 8s ago 5m 394M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (5m) 8s ago 5m 200M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (5m) 8s ago 5m 242M 4016M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:23:57.169 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (3m) 8s ago 4m 50.8M - 2.43.0 a07b618ecd1d 96d161cf02d4 2024-09-13T14:23:57.743 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:23:57.743 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:23:57.743 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:23:57.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:57.743 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 1, 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 14, 2024-09-13T14:23:57.744 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:23:57.745 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:23:57.745 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:23:57.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:57 smithi159 ceph-mon[33330]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:57 smithi159 ceph-mon[33330]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:57 smithi042 ceph-mon[40707]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:57 smithi042 ceph-mon[40707]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:57 smithi042 ceph-mon[29349]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:57 smithi042 ceph-mon[29349]: from='client.24779 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.243 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:23:58.243 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:23:58.243 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/2 daemons upgraded", 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:23:58.244 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:23:58.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:58 smithi159 ceph-mon[33330]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:23:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:58 smithi159 ceph-mon[33330]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:58 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/83658631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[40707]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:23:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[40707]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/83658631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[29349]: pgmap v157: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:23:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[29349]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:23:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:58 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/83658631' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:23:59.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:23:59 smithi159 ceph-mon[33330]: from='client.14904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:23:59 smithi042 ceph-mon[40707]: from='client.14904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:23:59 smithi042 ceph-mon[29349]: from='client.14904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:00.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:00 smithi159 ceph-mon[33330]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:00 smithi042 ceph-mon[40707]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:00 smithi042 ceph-mon[29349]: pgmap v158: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:03.079 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:02 smithi042 ceph-mon[40707]: pgmap v159: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:03.080 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:02 smithi042 ceph-mon[29349]: pgmap v159: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:03.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:02 smithi159 ceph-mon[33330]: pgmap v159: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2024-09-13T14:24:03.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:24:03] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-09-13T14:24:04.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:04] ENGINE Bus STOPPING 2024-09-13T14:24:04.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setuser ceph since I am not root 2024-09-13T14:24:04.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ignoring --setgroup ceph since I am not root 2024-09-13T14:24:04.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:04.532+0000 7ff314719040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T14:24:04.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:24:04.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: Activating manager daemon x 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: mgrmap e23: x(active, starting, since 0.0106559s) 2024-09-13T14:24:04.969 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:04.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[40707]: Manager daemon x is now available 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:04.971 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: Activating manager daemon x 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: mgrmap e23: x(active, starting, since 0.0106559s) 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:04.972 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:04.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:04.974 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:04.974 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-mon[29349]: Manager daemon x is now available 2024-09-13T14:24:04.974 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:04.624+0000 7ff314719040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T14:24:04.974 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:04.863+0000 7ff314719040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T14:24:04.974 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:04 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:04.965+0000 7ff314719040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T14:24:04.998 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:04] ENGINE Bus STOPPED 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: pgmap v160: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 172.21.15.42:0/4129043475' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: Activating manager daemon x 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.14442 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: mgrmap e23: x(active, starting, since 0.0106559s) 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:05.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:05.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:04 smithi159 ceph-mon[33330]: Manager daemon x is now available 2024-09-13T14:24:05.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:05 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:05.198+0000 7ff314719040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:24:05.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:05 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:05.411+0000 7ff314719040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T14:24:06.273 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:05 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:05.892+0000 7ff314719040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T14:24:06.273 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:06.000+0000 7ff314719040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T14:24:06.302 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:06] ENGINE Bus STARTING 2024-09-13T14:24:06.302 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:06] ENGINE Serving on http://:::9283 2024-09-13T14:24:06.302 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:24:06] ENGINE Bus STARTED 2024-09-13T14:24:06.302 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.299+0000 7fbc9584d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.302 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.299+0000 7fbc9584d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:06.272+0000 7ff314719040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T14:24:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: mgrmap e24: x(active, since 1.0153s) 2024-09-13T14:24:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: Checking for cert/key for grafana.a 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: Migrating grafana.a cert to cert store 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: Migrating grafana.a key to cert store 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: mgrmap e24: x(active, since 1.0153s) 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: Checking for cert/key for grafana.a 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: Migrating grafana.a cert to cert store 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: Migrating grafana.a key to cert store 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:06.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:06 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:06.603 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.299+0000 7fbc9584d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.299+0000 7fbc9584d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.299+0000 7fbc9584d640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.302+0000 7fbc99c15640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.302+0000 7fbc99c15640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.302+0000 7fbc99c15640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.302+0000 7fbc99c15640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:06.302+0000 7fbc99c15640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: mgrmap e24: x(active, since 1.0153s) 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.604 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: Checking for cert/key for grafana.a 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: Migrating grafana.a cert to cert store 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: Migrating grafana.a key to cert store 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:06.605 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:06.606 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:06 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[40707]: pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[40707]: Deploying cephadm binary to smithi042 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[40707]: Deploying cephadm binary to smithi159 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[29349]: pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[29349]: Deploying cephadm binary to smithi042 2024-09-13T14:24:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-mon[29349]: Deploying cephadm binary to smithi159 2024-09-13T14:24:07.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:07 smithi159 ceph-mon[33330]: pgmap v4: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:07.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:07 smithi159 ceph-mon[33330]: Deploying cephadm binary to smithi042 2024-09-13T14:24:07.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:07 smithi159 ceph-mon[33330]: Deploying cephadm binary to smithi159 2024-09-13T14:24:08.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:07.674+0000 7ff314719040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T14:24:08.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:07.772+0000 7ff314719040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T14:24:08.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:08 smithi042 ceph-mon[29349]: mgrmap e25: x(active, since 3s) 2024-09-13T14:24:08.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:08 smithi042 ceph-mon[40707]: mgrmap e25: x(active, since 3s) 2024-09-13T14:24:08.627 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-mon[33330]: mgrmap e25: x(active, since 3s) 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:08.625+0000 7fbc7eb20640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:24:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: Traceback (most recent call last): 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: self._connections.run(self.expiration_interval) 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: self._run(expiration_interval) 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:08.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: s = self.context.wrap_socket( 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: return self.sslsocket_class._create( 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: self.do_handshake() 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: self._sslobj.do_handshake() 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:09.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:09.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:08 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:08.942+0000 7ff314719040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T14:24:09.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.146+0000 7ff314719040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T14:24:09.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:08] ENGINE Bus STARTING 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:08] ENGINE Serving on https://172.21.15.159:7150 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: Traceback (most recent call last): 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._run(expiration_interval) 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: s = self.context.wrap_socket( 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: return self.sslsocket_class._create( 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: self.do_handshake() 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._sslobj.do_handshake() 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:08] ENGINE Serving on http://172.21.15.159:8765 2024-09-13T14:24:09.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:09 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:08] ENGINE Bus STARTED 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:08] ENGINE Bus STARTING 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:08] ENGINE Serving on https://172.21.15.159:7150 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:09.751 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: self.do_handshake() 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:08] ENGINE Serving on http://172.21.15.159:8765 2024-09-13T14:24:09.752 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:08] ENGINE Bus STARTED 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: pgmap v5: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:08] ENGINE Bus STARTING 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:08] ENGINE Serving on https://172.21.15.159:7150 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:08] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:09.753 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:09.754 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:08] ENGINE Serving on http://172.21.15.159:8765 2024-09-13T14:24:09.755 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:08] ENGINE Bus STARTED 2024-09-13T14:24:09.755 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.381+0000 7ff314719040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T14:24:09.755 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.487+0000 7ff314719040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T14:24:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.748+0000 7ff314719040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T14:24:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.866+0000 7ff314719040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T14:24:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:09.972+0000 7ff314719040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T14:24:10.380 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.261+0000 7ff314719040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T14:24:10.644 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-mon[29349]: mgrmap e26: x(active, since 5s) 2024-09-13T14:24:10.645 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.379+0000 7ff314719040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T14:24:10.645 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.494+0000 7ff314719040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T14:24:10.645 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.643+0000 7ff314719040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T14:24:10.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-mon[40707]: mgrmap e26: x(active, since 5s) 2024-09-13T14:24:10.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:10 smithi159 ceph-mon[33330]: mgrmap e26: x(active, since 5s) 2024-09-13T14:24:11.009 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.740+0000 7ff314719040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T14:24:11.010 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:10.844+0000 7ff314719040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T14:24:11.746 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:11 smithi159 ceph-mon[33330]: pgmap v6: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 95 op/s 2024-09-13T14:24:11.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:11 smithi042 ceph-mon[29349]: pgmap v6: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 95 op/s 2024-09-13T14:24:11.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:11 smithi042 ceph-mon[40707]: pgmap v6: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 95 op/s 2024-09-13T14:24:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[29349]: Standby manager daemon y started 2024-09-13T14:24:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:12.327+0000 7ff314719040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:24:12] ENGINE Bus STARTING 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: CherryPy Checker: 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: The Application mounted at '' has an empty config. 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: 2024-09-13T14:24:12.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:24:12] ENGINE Serving on http://:::9283 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: [13/Sep/2024:14:24:12] ENGINE Bus STARTED 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[40707]: Standby manager daemon y started 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:12.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:12 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:12.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:12 smithi159 ceph-mon[33330]: Standby manager daemon y started 2024-09-13T14:24:12.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:12 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:12.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:12 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:12.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:12 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:12.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:12 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/3722700013' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:13.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:24:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:13.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:13 smithi159 ceph-mon[33330]: pgmap v7: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 546 op/s 2024-09-13T14:24:13.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:13 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:13.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:13 smithi159 ceph-mon[33330]: mgrmap e27: x(active, since 8s), standbys: y 2024-09-13T14:24:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:13 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:13 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:13.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[29349]: pgmap v7: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 546 op/s 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[29349]: mgrmap e27: x(active, since 8s), standbys: y 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[40707]: pgmap v7: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 25 KiB/s rd, 439 B/s wr, 546 op/s 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[40707]: mgrmap e27: x(active, since 8s), standbys: y 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:13.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:13 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.969 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:14.970 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi042 to 2481M 2024-09-13T14:24:14.970 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:14 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi042 to 2481M 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:15.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi042 to 2481M 2024-09-13T14:24:15.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:14 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:15 smithi159 ceph-mon[33330]: pgmap v8: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 425 op/s 2024-09-13T14:24:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:15 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:15 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[40707]: pgmap v8: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 425 op/s 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[29349]: pgmap v8: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 20 KiB/s rd, 341 B/s wr, 425 op/s 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:15 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[40707]: pgmap v9: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-13T14:24:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[29349]: pgmap v9: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-13T14:24:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:17 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:17 smithi159 ceph-mon[33330]: pgmap v9: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 16 KiB/s rd, 279 B/s wr, 717 op/s 2024-09-13T14:24:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:17 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:17 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:19 smithi159 ceph-mon[33330]: pgmap v10: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-13T14:24:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:19 smithi042 ceph-mon[40707]: pgmap v10: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-13T14:24:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:19 smithi042 ceph-mon[29349]: pgmap v10: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 656 op/s 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 657 op/s 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:21 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 657 op/s 2024-09-13T14:24:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: pgmap v11: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 657 op/s 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T14:24:22.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:22.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:22.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:21 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:23.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[40707]: Adjusting osd_memory_target on smithi159 to 2480M 2024-09-13T14:24:23.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:24:23.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:24:23.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[29349]: Adjusting osd_memory_target on smithi159 to 2480M 2024-09-13T14:24:23.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:24:23.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:22 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:24:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:22 smithi159 ceph-mon[33330]: Adjusting osd_memory_target on smithi159 to 2480M 2024-09-13T14:24:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:22 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:24:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:22 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:24:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[29610]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:24:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:24:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: pgmap v12: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 989 op/s 2024-09-13T14:24:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: pgmap v12: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 989 op/s 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:23 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: pgmap v12: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 989 op/s 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:23 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:24 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:24 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:24 smithi159 ceph-mon[33330]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[40707]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:24:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:24 smithi042 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:24:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:25 smithi159 ceph-mon[33330]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:24:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:25 smithi159 ceph-mon[33330]: pgmap v13: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:24:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:25 smithi042 ceph-mon[40707]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:24:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:25 smithi042 ceph-mon[40707]: pgmap v13: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:24:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:25 smithi042 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:24:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:25 smithi042 ceph-mon[29349]: pgmap v13: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 710 op/s 2024-09-13T14:24:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:27 smithi159 ceph-mon[33330]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:24:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:27 smithi042 ceph-mon[40707]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:24:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:27 smithi042 ceph-mon[29349]: pgmap v14: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:24:28.498 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 systemd[1]: Stopping Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:24:28.764 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:24:29.089 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:24:29.089 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:24:29.089 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:24:29.089 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:24:29.089 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.764Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.765Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.765Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.765Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:24:29.090 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[66074]: ts=2024-09-13T14:24:28.765Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:24:29.091 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:28 smithi159 podman[77489]: 2024-09-13 14:24:28.775219245 +0000 UTC m=+0.144232601 container died 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:29.397 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 podman[77489]: 2024-09-13 14:24:29.194876509 +0000 UTC m=+0.563889854 container cleanup 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:29.398 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 bash[77489]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:24:29.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 podman[77504]: 2024-09-13 14:24:29.396814923 +0000 UTC m=+0.621486658 container remove 96d161cf02d4ce32fa9a97f02910fbd4dfc6d8311fb4e0c4a40ee7f6124b35e2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:29.832 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:24:29.832 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (3m) 17s ago 4m 27.7M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:24:29.832 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (4m) 14s ago 4m 93.6M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (3m) 17s ago 3m 18.1M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (3m) 14s ago 3m 16.5M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (48s) 14s ago 8m 498M - 19.3.0-4697-gd58f7306 fec07016a2b0 222e56781225 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:9283,8765 running (10m) 17s ago 10m 283M - 18.2.4-843-g2c01458a c129f5a2fd4e f7442a44ace3 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (10m) 17s ago 10m 48.2M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (9m) 14s ago 9m 40.8M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (8m) 17s ago 8m 44.3M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (4m) 17s ago 4m 15.7M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (4m) 14s ago 4m 15.4M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 17s ago 8m 363M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (8m) 17s ago 8m 246M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (7m) 17s ago 7m 309M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:24:29.833 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (7m) 17s ago 7m 163M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:24:29.834 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (6m) 14s ago 6m 360M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:24:29.834 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (6m) 14s ago 6m 404M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:24:29.834 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (6m) 14s ago 6m 205M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:24:29.834 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (5m) 14s ago 5m 247M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:24:29.834 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (3m) 14s ago 5m 53.0M - 2.43.0 a07b618ecd1d 96d161cf02d4 2024-09-13T14:24:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:29 smithi042 ceph-mon[40707]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:29 smithi042 ceph-mon[40707]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:29 smithi042 ceph-mon[29349]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:29 smithi042 ceph-mon[29349]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:30.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:29 smithi159 ceph-mon[33330]: pgmap v15: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:30.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:29 smithi159 ceph-mon[33330]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:30.150 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:24:30.151 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 systemd[1]: Stopped Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:24:30.151 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 2.201s CPU time. 2024-09-13T14:24:30.151 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:29 smithi159 systemd[1]: Starting Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:24:30.419 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 1, 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:24:30.420 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 14, 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:24:30.421 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:24:30.498 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:30 smithi159 podman[77578]: 2024-09-13 14:24:30.057081869 +0000 UTC m=+0.039887965 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:24:30.932 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:24:30.932 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:24:30.932 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:24:30.932 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:24:30.932 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:24:30.933 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/2 daemons upgraded", 2024-09-13T14:24:30.933 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T14:24:30.933 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:24:30.933 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[40707]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[40707]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2995837021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[29349]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[29349]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:30 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2995837021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:24:31.173 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:30 smithi159 ceph-mon[33330]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.173 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:30 smithi159 ceph-mon[33330]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:31.174 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:30 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2995837021' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:24:31.498 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 podman[77578]: 2024-09-13 14:24:31.173359447 +0000 UTC m=+1.156165529 container create 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:31.841 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 podman[77578]: 2024-09-13 14:24:31.510112879 +0000 UTC m=+1.492918957 container init 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:31.841 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 podman[77578]: 2024-09-13 14:24:31.516983364 +0000 UTC m=+1.499789442 container start 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:24:31.841 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.573Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:24:31.842 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.574Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:24:31.842 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.574Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi159 (none))" 2024-09-13T14:24:31.842 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.574Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:24:31.842 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.574Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:24:31.842 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.576Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.578Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.579Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.580Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.585Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.585Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.453µs 2024-09-13T14:24:31.843 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.585Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.587Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.604Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.605Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.605Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.88µs wal_replay_duration=19.965582ms wbl_replay_duration=152ns total_replay_duration=20.022994ms 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.608Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.608Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.608Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 bash[77578]: 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 systemd[1]: Started Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.640Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=31.655079ms db_storage=1.116µs remote_storage=1.636µs web_handler=730ns query_engine=949ns scrape=507.44µs scrape_sd=101.166µs notify=9.707µs notify_sd=9.637µs rules=30.453795ms tracing=8.011µs 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.640Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:24:31.844 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:24:31.640Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:31 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: pgmap v16: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:24:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: from='client.14949 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:24:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:31 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[40707]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[40707]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[29349]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:33.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:32 smithi042 ceph-mon[29349]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:33.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:32 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:24:33.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:32 smithi159 ceph-mon[33330]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:33.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:32 smithi159 ceph-mon[33330]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T14:24:34.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:33 smithi042 ceph-mon[40707]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:34.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:33 smithi042 ceph-mon[29349]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:33 smithi159 ceph-mon[33330]: pgmap v17: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: Upgrade: Updating mgr.y 2024-09-13T14:24:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[40707]: Deploying daemon mgr.y on smithi042 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: Upgrade: Updating mgr.y 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:35.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:34 smithi042 ceph-mon[29349]: Deploying daemon mgr.y on smithi042 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: Upgrade: Updating mgr.y 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:34 smithi159 ceph-mon[33330]: Deploying daemon mgr.y on smithi042 2024-09-13T14:24:36.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:35 smithi159 ceph-mon[33330]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:36.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:35 smithi042 ceph-mon[40707]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:35 smithi042 ceph-mon[29349]: pgmap v18: 89 pgs: 89 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:36.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:36 smithi042 systemd[1]: Stopping Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:24:36.837 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:36 smithi042 podman[89338]: 2024-09-13 14:24:36.510414209 +0000 UTC m=+0.214022475 container died f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T14:24:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:36 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:36 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:37.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 podman[89338]: 2024-09-13 14:24:37.031697253 +0000 UTC m=+0.735305523 container cleanup f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:24:37.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 bash[89338]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y 2024-09-13T14:24:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:36 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:37.689 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 podman[89354]: 2024-09-13 14:24:37.418417757 +0000 UTC m=+0.907387831 container remove f7442a44ace327e1f91acb9005f5b1bd45d132edff3d604be1044bc653acc180 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T14:24:37.689 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:24:38.038 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:37 smithi042 ceph-mon[40707]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:38.038 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:37 smithi042 ceph-mon[29349]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:38.039 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T14:24:38.039 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 systemd[1]: Stopped Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:24:38.039 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:37 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service: Consumed 53.081s CPU time. 2024-09-13T14:24:38.039 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 systemd[1]: Starting Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:24:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:37 smithi159 ceph-mon[33330]: pgmap v19: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:38.710 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 podman[89470]: 2024-09-13 14:24:38.26747466 +0000 UTC m=+0.034370259 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:24:38.710 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 podman[89470]: 2024-09-13 14:24:38.403714454 +0000 UTC m=+0.170610038 container create 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T14:24:38.710 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 podman[89470]: 2024-09-13 14:24:38.709207711 +0000 UTC m=+0.476103298 container init 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:24:39.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 podman[89470]: 2024-09-13 14:24:38.722546904 +0000 UTC m=+0.489442483 container start 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:24:39.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 bash[89470]: 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 2024-09-13T14:24:39.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:38 smithi042 systemd[1]: Started Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:24:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:40 smithi159 ceph-mon[33330]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:40 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:40 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:40 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:40.340 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[40707]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[29349]: pgmap v20: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:40.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:40 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:40.748 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:40 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:24:40] "GET /metrics HTTP/1.1" 200 36145 "" "Prometheus/2.43.0" 2024-09-13T14:24:42.278 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:42 smithi159 ceph-mon[33330]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:42 smithi042 ceph-mon[29349]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:42 smithi042 ceph-mon[40707]: pgmap v21: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:24:44.165 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:44 smithi159 ceph-mon[33330]: pgmap v22: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:44.274 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:44 smithi042 ceph-mon[40707]: pgmap v22: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:44.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:44 smithi042 ceph-mon[29349]: pgmap v22: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:24:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[29349]: pgmap v23: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[40707]: pgmap v23: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:45 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:45 smithi159 ceph-mon[33330]: pgmap v23: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-09-13T14:24:45.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:45 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:45 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:45 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:45 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:46 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:46 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:46 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:46 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:46 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:46 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:48.068 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-mon[29349]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-mon[40707]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:47] ENGINE Bus STARTING 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: CherryPy Checker: 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: The Application mounted at '' has an empty config. 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:47] ENGINE Serving on http://:::9283 2024-09-13T14:24:48.069 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:47] ENGINE Bus STARTED 2024-09-13T14:24:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:47 smithi159 ceph-mon[33330]: pgmap v24: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: Standby manager daemon y restarted 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: Standby manager daemon y started 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: Standby manager daemon y restarted 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: Standby manager daemon y started 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:48 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: Standby manager daemon y restarted 2024-09-13T14:24:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: Standby manager daemon y started 2024-09-13T14:24:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T14:24:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:24:49.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T14:24:49.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:48 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:24:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:49 smithi042 ceph-mon[40707]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:49 smithi042 ceph-mon[40707]: mgrmap e28: x(active, since 44s), standbys: y 2024-09-13T14:24:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:49 smithi042 ceph-mon[29349]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:49 smithi042 ceph-mon[29349]: mgrmap e28: x(active, since 44s), standbys: y 2024-09-13T14:24:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:49 smithi159 ceph-mon[33330]: pgmap v25: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:49 smithi159 ceph-mon[33330]: mgrmap e28: x(active, since 44s), standbys: y 2024-09-13T14:24:50.749 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:50 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:24:50] "GET /metrics HTTP/1.1" 200 36145 "" "Prometheus/2.43.0" 2024-09-13T14:24:52.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:51 smithi042 ceph-mon[40707]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:52.086 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:51 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:52.086 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:51 smithi042 ceph-mon[29349]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:52.086 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:51 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:51 smithi159 ceph-mon[33330]: pgmap v26: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 682 op/s 2024-09-13T14:24:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:51 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:53.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:24:53.176+0000 7fbd2bcde640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-13T14:24:53.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ignoring --setuser ceph since I am not root 2024-09-13T14:24:53.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ignoring --setgroup ceph since I am not root 2024-09-13T14:24:53.512 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:53] ENGINE Bus STOPPING 2024-09-13T14:24:53.513 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:24:53.513 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:53] ENGINE Bus STOPPED 2024-09-13T14:24:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:24:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: Activating manager daemon y 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24649 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: mgrmap e29: y(active, starting, since 0.0099463s) 2024-09-13T14:24:54.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:54.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:54.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:54.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:54.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:53 smithi159 ceph-mon[33330]: Manager daemon y is now available 2024-09-13T14:24:54.344 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:24:54.344 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: Activating manager daemon y 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24649 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: mgrmap e29: y(active, starting, since 0.0099463s) 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:54.345 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:54.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[29349]: Manager daemon y is now available 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: pgmap v27: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:54.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 172.21.15.159:0/1771025767' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: Activating manager daemon y 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24649 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: mgrmap e29: y(active, starting, since 0.0099463s) 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T14:24:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T14:24:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:53 smithi042 ceph-mon[40707]: Manager daemon y is now available 2024-09-13T14:24:54.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:54] ENGINE Bus STARTING 2024-09-13T14:24:54.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.533+0000 7f73bbaa7640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.533+0000 7f73bbaa7640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.533+0000 7f73bbaa7640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.533+0000 7f73bbaa7640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.533+0000 7f73bbaa7640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.548+0000 7f73b58e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.548+0000 7f73b58e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.548+0000 7f73b58e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.548+0000 7f73b58e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:54.548+0000 7f73b58e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:54] ENGINE Serving on http://:::9283 2024-09-13T14:24:54.599 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:24:54] ENGINE Bus STARTED 2024-09-13T14:24:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: mgrmap e30: y(active, since 1.01468s) 2024-09-13T14:24:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: pgmap v3: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: mgrmap e30: y(active, since 1.01468s) 2024-09-13T14:24:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: pgmap v3: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:55.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: mgrmap e30: y(active, since 1.01468s) 2024-09-13T14:24:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: pgmap v3: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:24:55.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:24:55.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:24:55.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T14:24:55.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T14:24:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[29349]: pgmap v4: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:56.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[40707]: pgmap v4: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:56 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:56.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:56 smithi159 ceph-mon[33330]: pgmap v4: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:56 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:56 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:24:57.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:57 smithi159 ceph-mon[33330]: mgrmap e31: y(active, since 3s) 2024-09-13T14:24:57.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:57 smithi042 ceph-mon[29349]: mgrmap e31: y(active, since 3s) 2024-09-13T14:24:57.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:57 smithi042 ceph-mon[40707]: mgrmap e31: y(active, since 3s) 2024-09-13T14:24:58.685 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-mon[40707]: pgmap v5: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:58.686 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-mon[29349]: pgmap v5: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:58.686 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:58.403+0000 7f739b7ed640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:24:58] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: Traceback (most recent call last): 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._connections.run(self.expiration_interval) 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._run(expiration_interval) 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: s = self.context.wrap_socket( 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:58.687 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: return self.sslsocket_class._create( 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self.do_handshake() 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._sslobj.do_handshake() 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:58.688 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:24:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:24:58.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:58 smithi159 ceph-mon[33330]: pgmap v5: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail 2024-09-13T14:24:59.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:58] ENGINE Bus STARTING 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:58] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:58] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:59.966 INFO:journalctl@ceph.mon.c.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: self.do_handshake() 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:58] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: [13/Sep/2024:14:24:58] ENGINE Bus STARTED 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[40707]: mgrmap e32: y(active, since 5s) 2024-09-13T14:24:59.967 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:58] ENGINE Bus STARTING 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:58] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:58] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:24:59.968 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:58] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: [13/Sep/2024:14:24:58] ENGINE Bus STARTED 2024-09-13T14:24:59.969 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:24:59 smithi042 ceph-mon[29349]: mgrmap e32: y(active, since 5s) 2024-09-13T14:24:59.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:58] ENGINE Bus STARTING 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:58] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:58] ENGINE Error in HTTPServer.serve 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: Traceback (most recent call last): 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._run(expiration_interval) 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:24:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: s = self.context.wrap_socket( 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: return self.sslsocket_class._create( 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: self.do_handshake() 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._sslobj.do_handshake() 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:58] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:25:00.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: [13/Sep/2024:14:24:58] ENGINE Bus STARTED 2024-09-13T14:25:00.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:24:59 smithi159 ceph-mon[33330]: mgrmap e32: y(active, since 5s) 2024-09-13T14:25:00.953 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:00 smithi159 ceph-mon[33330]: pgmap v6: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 43 op/s 2024-09-13T14:25:00.953 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:00 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:00.953 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:00 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[40707]: pgmap v6: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 43 op/s 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[29349]: pgmap v6: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 43 op/s 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:00 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:01.509 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:25:01.952 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:25:01] ENGINE Bus STARTING 2024-09-13T14:25:01.952 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: CherryPy Checker: 2024-09-13T14:25:01.953 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: The Application mounted at '' has an empty config. 2024-09-13T14:25:01.953 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: 2024-09-13T14:25:01.953 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:25:01] ENGINE Serving on http://:::9283 2024-09-13T14:25:01.953 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: [13/Sep/2024:14:25:01] ENGINE Bus STARTED 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: pgmap v7: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 293 B/s wr, 530 op/s 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='client.24760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: Standby manager daemon x started 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:02.595 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: pgmap v7: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 293 B/s wr, 530 op/s 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='client.24760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.597 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (4m) 2s ago 5m 26.2M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (5m) 0s ago 5m 93.7M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (4m) 2s ago 4m 18.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (4m) 0s ago 4m 16.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (81s) 0s ago 9m 144M - 19.3.0-4697-gd58f7306 fec07016a2b0 222e56781225 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (24s) 2s ago 10m 491M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (11m) 2s ago 11m 52.5M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (9m) 0s ago 9m 44.7M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (9m) 2s ago 9m 46.9M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (4m) 2s ago 4m 16.0M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (4m) 0s ago 4m 15.6M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 2s ago 8m 375M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:25:02.598 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (8m) 2s ago 8m 250M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (8m) 2s ago 8m 316M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (7m) 2s ago 7m 168M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (7m) 0s ago 7m 369M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (7m) 0s ago 6m 416M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (6m) 0s ago 6m 209M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (6m) 0s ago 6m 252M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:25:02.599 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (32s) 0s ago 5m 38.2M - 2.43.0 a07b618ecd1d 2e7322388f4b 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: pgmap v7: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 293 B/s wr, 530 op/s 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='client.24760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3608355323' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:02.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.199 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:25:03.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:03.201 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:25:03.201 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:25:03.201 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:03.201 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:25:03.201 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:25:03.763 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T14:25:03.764 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:25:03.764 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T14:25:03.764 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T14:25:03.764 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:25:03.764 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: mgrmap e33: y(active, since 9s), standbys: x 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/631601180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: mgrmap e33: y(active, since 9s), standbys: x 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/631601180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='client.24887 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='client.24893 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: mgrmap e33: y(active, since 9s), standbys: x 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/631601180' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:04 smithi159 ceph-mon[33330]: pgmap v8: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 20 KiB/s rd, 227 B/s wr, 412 op/s 2024-09-13T14:25:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:04 smithi159 ceph-mon[33330]: from='client.24778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:05.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:04 smithi042 ceph-mon[29349]: pgmap v8: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 20 KiB/s rd, 227 B/s wr, 412 op/s 2024-09-13T14:25:05.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:04 smithi042 ceph-mon[29349]: from='client.24778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:04 smithi042 ceph-mon[40707]: pgmap v8: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 20 KiB/s rd, 227 B/s wr, 412 op/s 2024-09-13T14:25:05.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:04 smithi042 ceph-mon[40707]: from='client.24778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:06.951 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:06 smithi159 ceph-mon[33330]: pgmap v9: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 16 KiB/s rd, 186 B/s wr, 694 op/s 2024-09-13T14:25:06.951 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:06 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:06.951 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:06 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:06.951 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:06 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:06.951 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:06 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[40707]: pgmap v9: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 16 KiB/s rd, 186 B/s wr, 694 op/s 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[29349]: pgmap v9: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 16 KiB/s rd, 186 B/s wr, 694 op/s 2024-09-13T14:25:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:06 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:07 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:25:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:07 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:25:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:07 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:07 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:07 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: pgmap v10: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: pgmap v10: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: pgmap v10: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:25:09.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:09 smithi159 ceph-mon[33330]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:25:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:09 smithi159 ceph-mon[33330]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:25:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[40707]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:25:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[40707]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:25:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[29349]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T14:25:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[29349]: Reconfiguring daemon prometheus.a on smithi159 2024-09-13T14:25:10.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:10.707 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[69553]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:25:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T14:25:10.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:10 smithi042 ceph-mon[40707]: pgmap v11: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:10.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:10 smithi042 ceph-mon[29349]: pgmap v11: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:11.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:10 smithi159 ceph-mon[33330]: pgmap v11: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 635 op/s 2024-09-13T14:25:12.714 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 systemd[1]: Stopping Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:12.989 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-mon[33330]: pgmap v12: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 969 op/s 2024-09-13T14:25:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:12 smithi042 ceph-mon[40707]: pgmap v12: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 969 op/s 2024-09-13T14:25:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:12 smithi042 ceph-mon[29349]: pgmap v12: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 15 KiB/s rd, 170 B/s wr, 969 op/s 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:25:13.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.989Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.990Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.990Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[77591]: ts=2024-09-13T14:25:12.990Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:25:13.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:12 smithi159 podman[80081]: 2024-09-13 14:25:12.997884508 +0000 UTC m=+0.172107335 container died 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:13.699 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 podman[80081]: 2024-09-13 14:25:13.40271521 +0000 UTC m=+0.576938028 container cleanup 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:13.700 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 bash[80081]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:25:13.951 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 podman[80099]: 2024-09-13 14:25:13.699034074 +0000 UTC m=+0.693264521 container remove 2e7322388f4bd46028124637b1f84859d2ec46edc216f87aec1932e25de369b1 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:14.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:25:14.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 systemd[1]: Stopped Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:14.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.553s CPU time. 2024-09-13T14:25:14.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:13 smithi159 systemd[1]: Starting Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:14.635 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 podman[80182]: 2024-09-13 14:25:14.205376501 +0000 UTC m=+0.034778716 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-13T14:25:14.635 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 podman[80182]: 2024-09-13 14:25:14.339289967 +0000 UTC m=+0.168692182 container create 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:14.635 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 podman[80182]: 2024-09-13 14:25:14.634168613 +0000 UTC m=+0.463570832 container init 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:15.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-mon[33330]: pgmap v13: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 podman[80182]: 2024-09-13 14:25:14.641811135 +0000 UTC m=+0.471213345 container start 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.692Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.693Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.693Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi159 (none))" 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.693Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:25:15.000 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.693Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.696Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.696Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.697Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.698Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.713Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.713Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=4.6µs 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.714Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.716Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 bash[80182]: 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.737Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T14:25:15.001 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 systemd[1]: Started Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.742Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.742Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.743Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=45.322µs wal_replay_duration=28.833329ms wbl_replay_duration=135ns total_replay_duration=29.112243ms 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.747Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.747Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.748Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.784Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=36.064638ms db_storage=1.365µs remote_storage=1.591µs web_handler=476ns query_engine=1.037µs scrape=3.482586ms scrape_sd=169.535µs notify=17.049µs notify_sd=17.077µs rules=31.775259ms tracing=8.075µs 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.784Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T14:25:15.002 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:25:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:25:14.784Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:25:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:14 smithi042 ceph-mon[29349]: pgmap v13: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:25:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:14 smithi042 ceph-mon[40707]: pgmap v13: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:15 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:16.162 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:15 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.163 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:15 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:16.163 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:15 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:16.163 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:15 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:17.062 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:16 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:17.062 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:16 smithi159 ceph-mon[33330]: pgmap v14: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:16 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:16 smithi042 ceph-mon[40707]: pgmap v14: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:16 smithi042 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:25:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:16 smithi042 ceph-mon[29349]: pgmap v14: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:18.060 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: Upgrade: Updating mgr.x 2024-09-13T14:25:18.061 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:25:18.061 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:25:18.061 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:18.061 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: Deploying daemon mgr.x on smithi159 2024-09-13T14:25:18.061 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:17 smithi159 ceph-mon[33330]: pgmap v15: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: Upgrade: Updating mgr.x 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: Deploying daemon mgr.x on smithi159 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[40707]: pgmap v15: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: Upgrade: Updating mgr.x 2024-09-13T14:25:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T14:25:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T14:25:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: Deploying daemon mgr.x on smithi159 2024-09-13T14:25:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:17 smithi042 ceph-mon[29349]: pgmap v15: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:19.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:19 smithi159 systemd[1]: Stopping Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:19.958 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:19 smithi159 podman[80563]: 2024-09-13 14:25:19.685366011 +0000 UTC m=+0.205744531 container died 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T14:25:20.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 podman[80563]: 2024-09-13 14:25:20.062200301 +0000 UTC m=+0.582578814 container cleanup 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T14:25:20.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 bash[80563]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x 2024-09-13T14:25:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:20 smithi042 ceph-mon[40707]: pgmap v16: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:20 smithi042 ceph-mon[29349]: pgmap v16: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:20.678 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:20 smithi159 ceph-mon[33330]: pgmap v16: 89 pgs: 89 active+clean; 3.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-09-13T14:25:20.678 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 podman[80579]: 2024-09-13 14:25:20.27236468 +0000 UTC m=+0.582643076 container remove 222e56781225ce45e0e00ab45f98571ecaa7bb157c75b26451757da43d1ea0d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:25:20.678 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:25:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T14:25:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 systemd[1]: Stopped Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:20.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:20 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service: Consumed 28.552s CPU time. 2024-09-13T14:25:21.415 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:21 smithi159 systemd[1]: Starting Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:21.748 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:21 smithi159 podman[80699]: 2024-09-13 14:25:21.322594933 +0000 UTC m=+0.030509309 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:25:21.749 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:21 smithi159 podman[80699]: 2024-09-13 14:25:21.46541671 +0000 UTC m=+0.173331077 container create 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:25:22.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:21 smithi159 podman[80699]: 2024-09-13 14:25:21.911186976 +0000 UTC m=+0.619101364 container init 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:25:22.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:21 smithi159 podman[80699]: 2024-09-13 14:25:21.922646204 +0000 UTC m=+0.630560577 container start 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T14:25:22.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:22 smithi159 bash[80699]: 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 2024-09-13T14:25:22.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:22 smithi159 systemd[1]: Started Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[40707]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[29349]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:22 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:22.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:22 smithi159 ceph-mon[33330]: pgmap v17: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:22 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:22 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:22 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:25:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:25:23] "GET /metrics HTTP/1.1" 200 36141 "" "Prometheus/2.43.0" 2024-09-13T14:25:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:23 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:23 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:23 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:23 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:23 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:23 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:24 smithi159 ceph-mon[33330]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:25:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:24 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:24 smithi042 ceph-mon[29349]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:25:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:24 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:24 smithi042 ceph-mon[40707]: pgmap v18: 89 pgs: 89 active+clean; 2.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:25:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:24 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:26.922 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:26 smithi159 ceph-mon[33330]: pgmap v19: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:26 smithi042 ceph-mon[29349]: pgmap v19: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:26 smithi042 ceph-mon[40707]: pgmap v19: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:28 smithi159 ceph-mon[33330]: pgmap v20: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:29.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:28 smithi042 ceph-mon[40707]: pgmap v20: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:29.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:28 smithi042 ceph-mon[29349]: pgmap v20: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: pgmap v21: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: Standby manager daemon x restarted 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:25:30] ENGINE Bus STARTING 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: CherryPy Checker: 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: The Application mounted at '' has an empty config. 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: 2024-09-13T14:25:31.000 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:25:30] ENGINE Serving on http://:::9283 2024-09-13T14:25:31.001 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:25:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:25:30] ENGINE Bus STARTED 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: pgmap v21: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: Standby manager daemon x restarted 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: pgmap v21: 89 pgs: 89 active+clean; 2.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 698 op/s 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: Standby manager daemon x restarted 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: Standby manager daemon x started 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T14:25:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T14:25:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:30 smithi042 ceph-mon[29349]: from='mgr.? 172.21.15.159:0/3341191185' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T14:25:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:32 smithi159 ceph-mon[33330]: pgmap v22: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:32 smithi159 ceph-mon[33330]: mgrmap e34: y(active, since 38s), standbys: x 2024-09-13T14:25:32.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:32.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[40707]: pgmap v22: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[40707]: mgrmap e34: y(active, since 38s), standbys: x 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[29349]: pgmap v22: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:32.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[29349]: mgrmap e34: y(active, since 38s), standbys: x 2024-09-13T14:25:32.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:32.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:33.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:25:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:25:33] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.43.0" 2024-09-13T14:25:34.315 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:25:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:34 smithi042 ceph-mon[40707]: pgmap v23: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-09-13T14:25:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:34 smithi042 ceph-mon[29349]: pgmap v23: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-09-13T14:25:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:34 smithi159 ceph-mon[33330]: pgmap v23: 89 pgs: 89 active+clean; 2.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s 2024-09-13T14:25:35.270 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:25:35.270 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (4m) 34s ago 5m 26.2M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (5m) 4s ago 5m 92.4M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (4m) 34s ago 4m 18.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (4m) 4s ago 4m 17.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (13s) 4s ago 9m 93.2M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (56s) 34s ago 11m 491M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (11m) 34s ago 11m 52.5M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (10m) 4s ago 10m 47.4M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (10m) 34s ago 10m 46.9M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (5m) 34s ago 5m 16.0M - 1.5.0 0da6a335fe13 bb0b63759fde 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (5m) 4s ago 5m 15.6M - 1.5.0 0da6a335fe13 8afb3ca7efde 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (9m) 34s ago 9m 375M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (9m) 34s ago 9m 250M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:25:35.271 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (8m) 34s ago 8m 316M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (8m) 34s ago 8m 168M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (7m) 4s ago 7m 374M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (7m) 4s ago 7m 418M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (7m) 4s ago 7m 212M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (6m) 4s ago 6m 255M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:25:35.272 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (21s) 4s ago 6m 41.4M - 2.43.0 a07b618ecd1d 1e3f911b877f 2024-09-13T14:25:35.580 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:35 smithi159 ceph-mon[33330]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:35 smithi042 ceph-mon[40707]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:35 smithi042 ceph-mon[29349]: from='client.15021 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:35.863 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:25:35.863 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:25:35.863 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:25:35.864 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:25:35.865 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:25:35.865 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:25:36.367 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:25:36.367 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:25:36.367 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:25:36.367 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:25:36.368 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='client.24917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: pgmap v24: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/3466387896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:36 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='client.24917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: pgmap v24: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/3466387896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='client.24917 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: pgmap v24: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/3466387896' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:25:36.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:25:36.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:36.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.853 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:36.854 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:36.854 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:36 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all mgr 2024-09-13T14:25:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all crash 2024-09-13T14:25:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='client.24935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all rgw 2024-09-13T14:25:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all nfs 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:25:37.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:37 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all mgr 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all crash 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='client.24935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all rgw 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all nfs 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all mgr 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all crash 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='client.24935 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all rgw 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all nfs 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:25:37.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:37 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:38.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:38 smithi159 ceph-mon[33330]: pgmap v25: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:38 smithi042 ceph-mon[29349]: pgmap v25: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:38.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:38 smithi042 ceph-mon[40707]: pgmap v25: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:39.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:39 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:39 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:39.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:39 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:40.348 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:40 smithi042 systemd[1]: Stopping Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:40.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[40707]: pgmap v26: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:40.781 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[40707]: Upgrade: Updating node-exporter.a (1/2) 2024-09-13T14:25:40.782 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[40707]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:25:40.782 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:40 smithi042 podman[98448]: 2024-09-13 14:25:40.45954205 +0000 UTC m=+0.154408031 container died bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:40.782 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[29349]: pgmap v26: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:40.782 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[29349]: Upgrade: Updating node-exporter.a (1/2) 2024-09-13T14:25:40.782 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:40 smithi042 ceph-mon[29349]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:25:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:40 smithi159 ceph-mon[33330]: pgmap v26: 89 pgs: 89 active+clean; 2.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 662 op/s 2024-09-13T14:25:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:40 smithi159 ceph-mon[33330]: Upgrade: Updating node-exporter.a (1/2) 2024-09-13T14:25:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:40 smithi159 ceph-mon[33330]: Deploying daemon node-exporter.a on smithi042 2024-09-13T14:25:41.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:40 smithi042 podman[98448]: 2024-09-13 14:25:40.971095762 +0000 UTC m=+0.665961741 container cleanup bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:41.098 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:40 smithi042 bash[98448]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T14:25:41.598 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 podman[98465]: 2024-09-13 14:25:41.198400812 +0000 UTC m=+0.738895923 container remove bb0b63759fdec94e7a914237519c349482ffc81ff22a17ef93919441f7fffb2e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:41.598 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:25:41.943 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T14:25:41.943 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 systemd[1]: Stopped Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:41.943 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.934s CPU time. 2024-09-13T14:25:42.326 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:41 smithi042 systemd[1]: Starting Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:42 smithi042 ceph-mon[40707]: pgmap v27: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:42 smithi042 ceph-mon[29349]: pgmap v27: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:42.598 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:42 smithi042 bash[98590]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:25:42.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:42 smithi159 ceph-mon[33330]: pgmap v27: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:25:43.078 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:42 smithi042 bash[98590]: Getting image source signatures 2024-09-13T14:25:43.078 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:42 smithi042 bash[98590]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:25:43.078 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:42 smithi042 bash[98590]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:25:43.079 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:42 smithi042 bash[98590]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:25:43.341 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:25:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:25:43] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-09-13T14:25:44.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:44 smithi042 ceph-mon[40707]: pgmap v28: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:25:44.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:44 smithi042 ceph-mon[29349]: pgmap v28: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:25:44.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:44 smithi159 ceph-mon[33330]: pgmap v28: 89 pgs: 89 active+clean; 1.7 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:25:46.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:46 smithi042 ceph-mon[40707]: pgmap v29: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:46.848 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:46 smithi042 bash[98590]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:25:46.848 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:46 smithi042 bash[98590]: Writing manifest to image destination 2024-09-13T14:25:46.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:46 smithi042 ceph-mon[29349]: pgmap v29: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:46.868 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:46 smithi159 ceph-mon[33330]: pgmap v29: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:25:47.273 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:46 smithi042 podman[98590]: 2024-09-13 14:25:46.985256141 +0000 UTC m=+4.857328227 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:25:47.274 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 podman[98590]: 2024-09-13 14:25:47.109872544 +0000 UTC m=+4.981944613 container create 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:47.733 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 podman[98590]: 2024-09-13 14:25:47.481402013 +0000 UTC m=+5.353474085 container init 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 podman[98590]: 2024-09-13 14:25:47.490708952 +0000 UTC m=+5.362781027 container start 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.495Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.495Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.496Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.496Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.496Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.496Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:25:47.734 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:25:47.735 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.497Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:25:47.736 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:25:47.737 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.498Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.499Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.499Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:25:47.738 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.499Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:25:47.739 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a[98670]: ts=2024-09-13T14:25:47.499Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:25:47.739 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 bash[98590]: 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 2024-09-13T14:25:47.739 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 14:25:47 smithi042 systemd[1]: Started Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:48.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:48 smithi159 ceph-mon[33330]: pgmap v30: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:48 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:48 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:48.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[40707]: pgmap v30: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:48.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:48.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:48.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[29349]: pgmap v30: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:48.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:48.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:48 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:50.659 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:50 smithi159 ceph-mon[33330]: pgmap v31: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:50 smithi042 ceph-mon[40707]: pgmap v31: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:50.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:50 smithi042 ceph-mon[29349]: pgmap v31: 89 pgs: 89 active+clean; 1.4 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-09-13T14:25:50.998 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:50 smithi159 systemd[1]: Stopping Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:51.498 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:51 smithi159 podman[84360]: 2024-09-13 14:25:51.217639352 +0000 UTC m=+0.262489083 container died 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:51 smithi042 ceph-mon[40707]: Upgrade: Updating node-exporter.b (2/2) 2024-09-13T14:25:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:51 smithi042 ceph-mon[40707]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:25:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:51 smithi042 ceph-mon[29349]: Upgrade: Updating node-exporter.b (2/2) 2024-09-13T14:25:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:51 smithi042 ceph-mon[29349]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:25:51.989 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:51 smithi159 podman[84360]: 2024-09-13 14:25:51.637044079 +0000 UTC m=+0.681893810 container cleanup 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:51.989 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:51 smithi159 bash[84360]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T14:25:51.990 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:51 smithi159 ceph-mon[33330]: Upgrade: Updating node-exporter.b (2/2) 2024-09-13T14:25:51.990 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:51 smithi159 ceph-mon[33330]: Deploying daemon node-exporter.b on smithi159 2024-09-13T14:25:52.248 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:51 smithi159 podman[84374]: 2024-09-13 14:25:51.989062288 +0000 UTC m=+0.767276834 container remove 8afb3ca7efde3baf2759d15e6c678adb3029eb916f1f23b1d2cf498f9496242e (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:52.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:51 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:25:52.749 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:52 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T14:25:52.749 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:52 smithi159 systemd[1]: Stopped Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:52.749 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:52 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.967s CPU time. 2024-09-13T14:25:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:52 smithi159 ceph-mon[33330]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:52.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:52 smithi042 ceph-mon[40707]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:52 smithi042 ceph-mon[29349]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:25:53.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:52 smithi159 systemd[1]: Starting Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:25:53.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:25:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:25:53] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-09-13T14:25:53.748 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:53 smithi159 bash[84490]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-13T14:25:54.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:53 smithi159 bash[84490]: Getting image source signatures 2024-09-13T14:25:54.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:53 smithi159 bash[84490]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-13T14:25:54.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:53 smithi159 bash[84490]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-13T14:25:54.249 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:53 smithi159 bash[84490]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-13T14:25:54.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:54 smithi042 ceph-mon[40707]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:25:54.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:54 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:54.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:54 smithi042 ceph-mon[29349]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:25:54.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:54 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:54 smithi159 ceph-mon[33330]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:25:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:54 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:25:55.748 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:55 smithi159 bash[84490]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-13T14:25:55.749 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:55 smithi159 bash[84490]: Writing manifest to image destination 2024-09-13T14:25:56.168 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:55 smithi159 podman[84490]: 2024-09-13 14:25:55.867536227 +0000 UTC m=+2.750740838 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-13T14:25:56.168 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:55 smithi159 podman[84490]: 2024-09-13 14:25:55.998914044 +0000 UTC m=+2.882118644 container create 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:56.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-mon[33330]: pgmap v34: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:56.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:56.751 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 podman[84490]: 2024-09-13 14:25:56.310649413 +0000 UTC m=+3.193854016 container init 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:56.751 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 podman[84490]: 2024-09-13 14:25:56.320007469 +0000 UTC m=+3.203212072 container start 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.323Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.323Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.325Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.325Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.325Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.325Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T14:25:56.752 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T14:25:56.753 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.326Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T14:25:56.754 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T14:25:56.755 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T14:25:56.756 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T14:25:56.756 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.327Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.328Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b[84559]: ts=2024-09-13T14:25:56.328Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 bash[84490]: 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a 2024-09-13T14:25:56.757 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 14:25:56 smithi159 systemd[1]: Started Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:25:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[29349]: pgmap v34: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:56.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[40707]: pgmap v34: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:25:56.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:56 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:25:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:25:58 smithi159 ceph-mon[33330]: pgmap v35: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:25:59.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:25:58 smithi042 ceph-mon[40707]: pgmap v35: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:25:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:25:58 smithi042 ceph-mon[29349]: pgmap v35: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:26:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:00 smithi159 ceph-mon[33330]: pgmap v36: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:26:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:00 smithi042 ceph-mon[29349]: pgmap v36: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:26:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:00 smithi042 ceph-mon[40707]: pgmap v36: 89 pgs: 89 active+clean; 937 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-13T14:26:02.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:02 smithi159 ceph-mon[33330]: pgmap v37: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:26:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[40707]: pgmap v37: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:26:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[29349]: pgmap v37: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:26:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.331 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:03] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.43.0" 2024-09-13T14:26:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[40707]: pgmap v38: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:26:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:04.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[29349]: pgmap v38: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:26:04.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:04.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:04 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:04 smithi159 ceph-mon[33330]: pgmap v38: 89 pgs: 89 active+clean; 672 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 715 op/s 2024-09-13T14:26:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:04 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:04 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:06.899 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:26:06.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:06 smithi159 ceph-mon[33330]: pgmap v39: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:26:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:06 smithi042 ceph-mon[40707]: pgmap v39: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:26:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:06 smithi042 ceph-mon[29349]: pgmap v39: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:26:07.899 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:26:07.899 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (5m) 6s ago 6m 26.2M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:26:07.899 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (6m) 5s ago 6m 96.0M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (5m) 6s ago 5m 19.1M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (5m) 5s ago 5m 17.1M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (46s) 5s ago 10m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (89s) 6s ago 11m 510M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (12m) 6s ago 12m 52.3M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (10m) 5s ago 10m 44.0M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (10m) 6s ago 10m 45.5M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (20s) 6s ago 5m 7822k - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (12s) 5s ago 5m 5343k - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (10m) 6s ago 10m 382M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:26:07.900 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (9m) 6s ago 9m 255M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (9m) 6s ago 9m 325M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (8m) 6s ago 8m 172M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (8m) 5s ago 8m 378M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (8m) 5s ago 8m 421M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (7m) 5s ago 7m 214M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (7m) 5s ago 7m 258M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:26:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (53s) 5s ago 6m 44.5M - 2.43.0 a07b618ecd1d 1e3f911b877f 2024-09-13T14:26:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[29349]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[29349]: pgmap v40: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[29349]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[40707]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[40707]: pgmap v40: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:08 smithi042 ceph-mon[40707]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:08 smithi159 ceph-mon[33330]: from='client.15048 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:08 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:08 smithi159 ceph-mon[33330]: pgmap v40: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:08 smithi159 ceph-mon[33330]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:26:08.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:26:08.534 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:26:09.051 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:26:09.051 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:26:09.052 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/4199769605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/4199769605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='client.15057 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/4199769605' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:10.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: pgmap v41: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: pgmap v41: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:10.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='client.24953 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: pgmap v41: 89 pgs: 89 active+clean; 416 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 693 op/s 2024-09-13T14:26:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:12.331 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 132512 objects 2024-09-13T14:26:12.331 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :156.649 2024-09-13T14:26:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[29349]: Upgrade: Updating prometheus.a 2024-09-13T14:26:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[29349]: pgmap v42: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:26:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[29349]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:26:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[40707]: Upgrade: Updating prometheus.a 2024-09-13T14:26:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[40707]: pgmap v42: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:26:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:12 smithi042 ceph-mon[40707]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:26:12.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:12 smithi159 ceph-mon[33330]: Upgrade: Updating prometheus.a 2024-09-13T14:26:12.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:12 smithi159 ceph-mon[33330]: pgmap v42: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:26:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:12 smithi159 ceph-mon[33330]: Deploying daemon prometheus.a on smithi159 2024-09-13T14:26:13.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:13] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.43.0" 2024-09-13T14:26:14.548 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:14 smithi159 ceph-mon[33330]: pgmap v43: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:26:14.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:14 smithi042 ceph-mon[29349]: pgmap v43: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:26:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:14 smithi042 ceph-mon[40707]: pgmap v43: 89 pgs: 89 active+clean; 157 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T14:26:15.740 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-13T14:26:15.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-13T14:26:15.963 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:16 smithi042 ceph-mon[29349]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:26:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:16 smithi042 ceph-mon[40707]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:26:16.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:16 smithi159 ceph-mon[33330]: pgmap v44: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-13T14:26:18.294 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_0' removed 2024-09-13T14:26:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[29349]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-09-13T14:26:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1331534518' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[40707]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-09-13T14:26:18.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1331534518' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:18 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:18 smithi159 ceph-mon[33330]: pgmap v45: 89 pgs: 89 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 554 op/s 2024-09-13T14:26:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:18 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1331534518' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:18 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-09-13T14:26:18.991 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:26:18.991 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:26:18.992 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-13T14:26:18.992 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-13T14:26:19.198 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:19.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:19 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:26:19.581 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:19 smithi042 ceph-mon[29349]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:19.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:19 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:26:19.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:19 smithi042 ceph-mon[40707]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:19.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:19 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:26:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:19 smithi159 ceph-mon[33330]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T14:26:20.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:20 smithi042 ceph-mon[29349]: pgmap v47: 73 pgs: 73 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:20 smithi042 ceph-mon[40707]: pgmap v47: 73 pgs: 73 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:20.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:20 smithi159 ceph-mon[33330]: pgmap v47: 73 pgs: 73 active+clean; 581 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:21.666 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_1' created 2024-09-13T14:26:21.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:21 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-13T14:26:21.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:21 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-13T14:26:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:21 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-09-13T14:26:22.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-13T14:26:22.620 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:26:22.680 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:22 smithi159 ceph-mon[33330]: pgmap v48: 73 pgs: 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:22.680 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:22 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:26:22.680 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:22 smithi159 ceph-mon[33330]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:22.680 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:22 smithi159 ceph-mon[33330]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:23.021 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[29349]: pgmap v48: 73 pgs: 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[29349]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[29349]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[40707]: pgmap v48: 73 pgs: 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/1919150848' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[40707]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T14:26:23.022 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:22 smithi042 ceph-mon[40707]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T14:26:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:23] "GET /metrics HTTP/1.1" 200 36134 "" "Prometheus/2.43.0" 2024-09-13T14:26:24.499 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 systemd[1]: Stopping Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:26:24.787 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: pgmap v51: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:24.787 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:24.787 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:24.787 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:24.787 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:24.788 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2025523181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:24.788 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.504Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T14:26:24.788 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[80194]: ts=2024-09-13T14:26:24.505Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T14:26:24.789 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 podman[88441]: 2024-09-13 14:26:24.515341565 +0000 UTC m=+0.134096202 container died 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: pgmap v51: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2025523181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: pgmap v51: 89 pgs: 16 unknown, 73 active+clean; 581 KiB data, 1016 MiB used, 714 GiB / 715 GiB avail 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2025523181' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:25.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:24 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T14:26:25.129 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 podman[88441]: 2024-09-13 14:26:24.877379045 +0000 UTC m=+0.496133675 container cleanup 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:25.129 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:24 smithi159 bash[88441]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T14:26:25.337 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-13T14:26:25.480 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 podman[88457]: 2024-09-13 14:26:25.128702776 +0000 UTC m=+0.615252060 container remove 1e3f911b877f16eee49b82618f71c28dcdaaebeb066cbd9a1d39d7d7e892d0d7 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:25.480 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T14:26:25.480 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 systemd[1]: Stopped Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:26:25.480 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.393s CPU time. 2024-09-13T14:26:25.739 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 systemd[1]: Starting Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:26:25.739 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 podman[88591]: 2024-09-13 14:26:25.58947597 +0000 UTC m=+0.016746779 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-13T14:26:25.739 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 podman[88591]: 2024-09-13 14:26:25.73925567 +0000 UTC m=+0.166526513 container create e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:26.249 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 podman[88591]: 2024-09-13 14:26:25.990600596 +0000 UTC m=+0.417871396 container init e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:25 smithi159 podman[88591]: 2024-09-13 14:26:25.995092482 +0000 UTC m=+0.422363280 container start e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.015Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.015Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.015Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi159 (none))" 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.015Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.015Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.017Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.017Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.019Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T14:26:26.250 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.019Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.020Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.020Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.567µs 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.020Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.021Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.028Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.029Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.030Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.030Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.030Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.999µs wal_replay_duration=9.851561ms wbl_replay_duration=125ns total_replay_duration=9.894681ms 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.032Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.032Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-13T14:26:26.251 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.032Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T14:26:26.252 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.048Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.56521ms db_storage=899ns remote_storage=1.469µs web_handler=364ns query_engine=558ns scrape=403.318µs scrape_sd=90.52µs notify=9.72µs notify_sd=8.882µs rules=14.716836ms tracing=6.354µs 2024-09-13T14:26:26.252 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.048Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-13T14:26:26.252 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T14:26:26.048Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T14:26:26.252 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 bash[88591]: e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e 2024-09-13T14:26:26.252 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 14:26:26 smithi159 systemd[1]: Started Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:26:26.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: pgmap v53: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:26:26.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:26.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:26.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: pgmap v53: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:26 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: pgmap v53: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:26 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:27.056 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-13T14:26:27.059 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:26:27.060 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:26:27.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:26:27.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:26:27.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_102613 2024-09-13T14:26:27.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:26:27.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:26:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:27 smithi159 ceph-mon[33330]: mgrmap e35: y(active, since 93s), standbys: x 2024-09-13T14:26:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:27 smithi042 ceph-mon[29349]: mgrmap e35: y(active, since 93s), standbys: x 2024-09-13T14:26:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:27 smithi042 ceph-mon[40707]: mgrmap e35: y(active, since 93s), standbys: x 2024-09-13T14:26:28.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 4059 4043 252.672 252.688 0.00481231 0.0039422 2024-09-13T14:26:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:28 smithi159 ceph-mon[33330]: pgmap v55: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 368 B/s wr, 0 op/s 2024-09-13T14:26:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:28 smithi159 ceph-mon[33330]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:28 smithi159 ceph-mon[33330]: Cluster is now healthy 2024-09-13T14:26:29.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[29349]: pgmap v55: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 368 B/s wr, 0 op/s 2024-09-13T14:26:29.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[29349]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:29.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[29349]: Cluster is now healthy 2024-09-13T14:26:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[40707]: pgmap v55: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 368 B/s wr, 0 op/s 2024-09-13T14:26:29.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[40707]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:26:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:28 smithi042 ceph-mon[40707]: Cluster is now healthy 2024-09-13T14:26:29.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 7875 7859 245.574 238.5 0.00354614 0.00406404 2024-09-13T14:26:30.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 11424 11408 237.647 221.812 0.00357293 0.00419926 2024-09-13T14:26:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:30 smithi159 ceph-mon[33330]: pgmap v56: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 312 B/s wr, 0 op/s 2024-09-13T14:26:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:30 smithi042 ceph-mon[29349]: pgmap v56: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 312 B/s wr, 0 op/s 2024-09-13T14:26:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:30 smithi042 ceph-mon[40707]: pgmap v56: 89 pgs: 89 active+clean; 581 KiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 312 B/s wr, 0 op/s 2024-09-13T14:26:31.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 14728 14712 229.856 206.5 0.00943269 0.00434393 2024-09-13T14:26:32.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 17756 17740 221.731 189.25 0.0053328 0.00450298 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[40707]: pgmap v57: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[29349]: pgmap v57: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:32.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:32 smithi159 ceph-mon[33330]: pgmap v57: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:26:32.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.119 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 20660 20644 215.023 181.5 0.00610473 0.00464495 2024-09-13T14:26:33.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:33 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:33 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:33 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:33 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:33 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:33.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:33 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:34.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 23420 23404 208.946 172.5 0.00720276 0.00478091 2024-09-13T14:26:34.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:34 smithi159 ceph-mon[33330]: pgmap v58: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-13T14:26:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:34 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:34.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:34 smithi042 ceph-mon[40707]: pgmap v58: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-13T14:26:34.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:34 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:34.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:34 smithi042 ceph-mon[29349]: pgmap v58: 89 pgs: 89 active+clean; 543 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2024-09-13T14:26:34.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:34 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:35.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 26164 26148 204.263 171.5 0.00500258 0.00489011 2024-09-13T14:26:36.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 28704 28688 199.205 158.75 0.00312392 0.00501438 2024-09-13T14:26:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:36 smithi159 ceph-mon[33330]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-13T14:26:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:36 smithi042 ceph-mon[29349]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-13T14:26:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:36 smithi042 ceph-mon[40707]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-09-13T14:26:37.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 31033 31017 193.839 145.562 0.00456952 0.00515385 2024-09-13T14:26:38.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 33349 33333 189.374 144.75 0.0023863 0.00526552 2024-09-13T14:26:38.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:38 smithi159 ceph-mon[33330]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:26:38.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:38 smithi042 ceph-mon[29349]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:26:38.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:38 smithi042 ceph-mon[40707]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T14:26:39.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 35268 35252 183.587 119.938 0.00886904 0.0054423 2024-09-13T14:26:39.677 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:26:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:39] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-09-13T14:26:40.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 37009 36993 177.833 108.812 0.00748065 0.00561779 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:40 smithi159 ceph-mon[33330]: from='client.24847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:40.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[29349]: from='client.24847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:40.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:40 smithi042 ceph-mon[40707]: from='client.24847 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:40.795 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:26:40.795 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (6m) 39s ago 6m 26.2M - 0.25.0 c8568f914cd2 4ea561b43f2e 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (6m) 9s ago 6m 90.4M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (5m) 39s ago 5m 19.1M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (5m) 9s ago 5m 17.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (79s) 9s ago 10m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (2m) 39s ago 12m 510M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (12m) 39s ago 12m 52.3M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (11m) 9s ago 11m 44.5M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (11m) 39s ago 11m 45.5M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (53s) 39s ago 6m 7822k - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (44s) 9s ago 6m 8010k - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (10m) 39s ago 10m 382M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:26:40.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (10m) 39s ago 10m 255M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (9m) 39s ago 9m 325M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (9m) 39s ago 9m 172M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (9m) 9s ago 9m 292M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (8m) 9s ago 8m 315M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (8m) 9s ago 8m 174M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (7m) 9s ago 7m 209M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:26:40.797 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (15s) 9s ago 7m 32.9M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:26:41.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 38641 38625 172.416 102 0.0160944 0.00579503 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:41.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:26:41.393 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:26:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:41 smithi159 ceph-mon[33330]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:41 smithi159 ceph-mon[33330]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.513 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:41 smithi042 ceph-mon[29349]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.513 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:41 smithi042 ceph-mon[29349]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.513 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:41 smithi042 ceph-mon[40707]: from='client.24989 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.513 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:41 smithi042 ceph-mon[40707]: from='client.15111 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:41.973 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:26:41.973 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:26:41.973 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:26:41.973 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:26:41.974 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:26:41.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T14:26:41.974 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:26:41.974 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T14:26:41.974 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading alertmanager daemons", 2024-09-13T14:26:41.975 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:26:41.975 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:26:42.120 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 40228 40212 167.533 99.1875 0.0115009 0.00596364 2024-09-13T14:26:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:42 smithi042 ceph-mon[29349]: pgmap v62: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-09-13T14:26:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:42 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/2949336131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:42 smithi042 ceph-mon[40707]: pgmap v62: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-09-13T14:26:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:42 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2949336131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:42.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:42 smithi159 ceph-mon[33330]: pgmap v62: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-09-13T14:26:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:42 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2949336131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:26:43.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 41923 41907 163.683 105.938 0.0103642 0.00610367 2024-09-13T14:26:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[40707]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[40707]: Upgrade: Updating alertmanager.a 2024-09-13T14:26:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[40707]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:26:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[29349]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[29349]: Upgrade: Updating alertmanager.a 2024-09-13T14:26:43.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:43 smithi042 ceph-mon[29349]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:26:43.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:43 smithi159 ceph-mon[33330]: from='client.15123 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:26:43.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:43 smithi159 ceph-mon[33330]: Upgrade: Updating alertmanager.a 2024-09-13T14:26:43.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:43 smithi159 ceph-mon[33330]: Deploying daemon alertmanager.a on smithi042 2024-09-13T14:26:44.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 43666 43650 160.462 108.938 0.00964275 0.00622612 2024-09-13T14:26:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:44 smithi042 ceph-mon[40707]: pgmap v63: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-13T14:26:44.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:44 smithi042 ceph-mon[29349]: pgmap v63: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-13T14:26:44.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:44 smithi159 ceph-mon[33330]: pgmap v63: 89 pgs: 89 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s 2024-09-13T14:26:45.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 45588 45572 158.218 120.125 0.0149452 0.0063146 2024-09-13T14:26:46.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 47377 47361 155.775 111.812 0.0105601 0.00641464 2024-09-13T14:26:46.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:46 smithi042 ceph-mon[40707]: pgmap v64: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 185 MiB/s wr, 2.95k op/s 2024-09-13T14:26:46.603 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:46 smithi042 ceph-mon[29349]: pgmap v64: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 185 MiB/s wr, 2.95k op/s 2024-09-13T14:26:46.614 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:46 smithi159 ceph-mon[33330]: pgmap v64: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 185 MiB/s wr, 2.95k op/s 2024-09-13T14:26:47.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:26:47.121520+0000 min lat: 0.00114625 max lat: 0.127575 avg lat: 0.00652066 2024-09-13T14:26:47.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:26:47.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 49056 49040 153.233 104.938 0.00749229 0.00652066 2024-09-13T14:26:48.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 50736 50720 150.936 105 0.00705371 0.00662047 2024-09-13T14:26:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:48 smithi042 ceph-mon[40707]: pgmap v65: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:48 smithi042 ceph-mon[29349]: pgmap v65: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:48.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:48 smithi159 ceph-mon[33330]: pgmap v65: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:49.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 52545 52529 149.214 113.062 0.00912749 0.00669688 2024-09-13T14:26:49.836 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:49] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-09-13T14:26:50.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 54310 54294 147.522 110.312 0.00332349 0.00677323 2024-09-13T14:26:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:50 smithi042 ceph-mon[40707]: pgmap v66: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:50 smithi042 ceph-mon[29349]: pgmap v66: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:50.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:50 smithi159 ceph-mon[33330]: pgmap v66: 89 pgs: 89 active+clean; 2.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.67k op/s 2024-09-13T14:26:51.121 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 56045 56029 145.893 108.438 0.00793155 0.00684907 2024-09-13T14:26:52.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 57734 57718 144.28 105.562 0.0052603 0.00692589 2024-09-13T14:26:52.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:52 smithi042 ceph-mon[40707]: pgmap v67: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T14:26:52.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:52 smithi042 ceph-mon[29349]: pgmap v67: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T14:26:52.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:52 smithi159 ceph-mon[33330]: pgmap v67: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T14:26:53.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 59305 59289 142.507 98.1875 0.00703656 0.00701108 2024-09-13T14:26:53.348 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:52 smithi042 systemd[1]: Stopping Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:26:53.825 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[76976]: ts=2024-09-13T14:26:53.475Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T14:26:53.826 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:53 smithi042 podman[103325]: 2024-09-13 14:26:53.529541377 +0000 UTC m=+0.301429659 container died 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:54.098 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:54 smithi042 podman[103325]: 2024-09-13 14:26:54.055611576 +0000 UTC m=+0.827499855 container cleanup 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:54.098 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:54 smithi042 bash[103325]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T14:26:54.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 60884 60868 140.883 98.6875 0.00983139 0.00709296 2024-09-13T14:26:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:54 smithi042 ceph-mon[40707]: pgmap v68: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:54.598 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:54 smithi042 podman[103343]: 2024-09-13 14:26:54.400364508 +0000 UTC m=+0.871589066 container remove 4ea561b43f2e01bf6b96bd84207ac5d6c202d1824381fa1a3f903ba1e070a1ad (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:54.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:54 smithi042 ceph-mon[29349]: pgmap v68: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:54.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:54 smithi159 ceph-mon[33330]: pgmap v68: 89 pgs: 89 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T14:26:55.098 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:54 smithi042 podman[103343]: 2024-09-13 14:26:54.743352435 +0000 UTC m=+1.214576994 volume remove c833daae139db6c12c7c3a72056faa5a51ea30e98cfe42a199db44aabb49db8f 2024-09-13T14:26:55.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 62377 62361 139.184 93.3125 0.0068312 0.00717958 2024-09-13T14:26:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:55 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:55 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:55.599 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:55 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T14:26:55.599 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:55 smithi042 systemd[1]: Stopped Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:26:55.599 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:55 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.737s CPU time. 2024-09-13T14:26:55.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:55 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:26:56.052 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:55 smithi042 systemd[1]: Starting Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:26:56.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 63832 63816 137.52 90.9375 0.0191568 0.00726563 2024-09-13T14:26:56.343 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 podman[103456]: 2024-09-13 14:26:55.957835148 +0000 UTC m=+0.029161502 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-13T14:26:56.343 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 podman[103456]: 2024-09-13 14:26:56.103115593 +0000 UTC m=+0.174441929 volume create 546391c59637fc7eff33da18f429f6b660a449f7e3bbddbda0611321289a4d50 2024-09-13T14:26:56.343 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 podman[103456]: 2024-09-13 14:26:56.212199365 +0000 UTC m=+0.283525702 container create 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-mon[29349]: pgmap v69: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-mon[40707]: pgmap v69: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:56.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:56 smithi159 ceph-mon[33330]: pgmap v69: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-13T14:26:57.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 65315 65299 136.026 92.6875 0.00683487 0.00734602 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 podman[103456]: 2024-09-13 14:26:56.874529468 +0000 UTC m=+0.945855809 container init 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 podman[103456]: 2024-09-13 14:26:56.884076496 +0000 UTC m=+0.955402833 container start 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.912Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.913Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.914Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.42 port=9094 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.917Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.958Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:26:57.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.959Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T14:26:57.350 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.964Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-13T14:26:57.350 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:56.964Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T14:26:57.350 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:57 smithi042 bash[103456]: 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 2024-09-13T14:26:57.350 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:57 smithi042 systemd[1]: Started Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:26:58.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 66669 66653 134.366 84.625 0.00501686 0.00743725 2024-09-13T14:26:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:26:58 smithi159 ceph-mon[33330]: pgmap v70: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:26:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:58.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[29349]: pgmap v70: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:26:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:26:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:26:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-mon[40707]: pgmap v70: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:26:59.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 68089 68073 132.94 88.75 0.00657583 0.00751642 2024-09-13T14:26:59.194 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:26:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:26:58.918Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001135792s 2024-09-13T14:27:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:26:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:26:59] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-09-13T14:27:00.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 69450 69434 131.489 85.0625 0.0120837 0.0075998 2024-09-13T14:27:00.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:00 smithi159 ceph-mon[33330]: pgmap v71: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:27:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:00 smithi042 ceph-mon[29349]: pgmap v71: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:27:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:00 smithi042 ceph-mon[40707]: pgmap v71: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-13T14:27:01.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 70867 70851 130.226 88.5625 0.0204995 0.00767333 2024-09-13T14:27:02.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 72236 72220 128.95 85.5625 0.0161415 0.00774979 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[29349]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[40707]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:02.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:02 smithi159 ceph-mon[33330]: pgmap v72: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T14:27:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 73440 73424 127.458 75.25 0.00701235 0.00784036 2024-09-13T14:27:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:03 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:03 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:04.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 74720 74704 126.175 80 0.018336 0.00792023 2024-09-13T14:27:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:04 smithi042 ceph-mon[29349]: pgmap v73: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:27:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:04 smithi042 ceph-mon[40707]: pgmap v73: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:27:04.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:04 smithi159 ceph-mon[33330]: pgmap v73: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-13T14:27:05.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 76129 76113 125.172 88.0625 0.00677242 0.00798385 2024-09-13T14:27:06.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 77442 77426 124.066 82.0625 0.00879053 0.00805498 2024-09-13T14:27:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:06 smithi042 ceph-mon[29349]: pgmap v74: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:27:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:06 smithi042 ceph-mon[40707]: pgmap v74: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:27:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:06 smithi159 ceph-mon[33330]: pgmap v74: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T14:27:07.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:27:07.123702+0000 min lat: 0.00114625 max lat: 0.127575 avg lat: 0.00812865 2024-09-13T14:27:07.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:27:07.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 78709 78693 122.944 79.1875 0.0128589 0.00812865 2024-09-13T14:27:07.229 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:27:06 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:27:06.925Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.008300379s 2024-09-13T14:27:08.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 79991 79975 121.9 80.125 0.0115921 0.00819866 2024-09-13T14:27:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:08 smithi042 ceph-mon[29349]: pgmap v75: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:08.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:08 smithi042 ceph-mon[40707]: pgmap v75: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:08 smithi159 ceph-mon[33330]: pgmap v75: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:09.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 81243 81227 120.86 78.25 0.00757067 0.00826815 2024-09-13T14:27:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:09 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:09] "GET /metrics HTTP/1.1" 200 36223 "" "Prometheus/2.51.0" 2024-09-13T14:27:09.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:09 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:09.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:09 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:10.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 82494 82478 119.868 78.1875 0.0122984 0.00833751 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: pgmap v76: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: pgmap v76: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.852 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: pgmap v76: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:27:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:11.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 83759 83743 118.94 79.0625 0.0103072 0.00840255 2024-09-13T14:27:11.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:11 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:11.802 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:11 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:11.803 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:11 smithi042 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T14:27:12.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 84886 84870 117.862 70.4375 0.0221053 0.00847874 2024-09-13T14:27:12.598 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:27:12.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:12 smithi042 ceph-mon[40707]: pgmap v77: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:27:12.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:12 smithi042 ceph-mon[29349]: pgmap v77: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:27:12.919 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:12 smithi159 ceph-mon[33330]: pgmap v77: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-09-13T14:27:13.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 85842 85826 116.599 59.75 0.0176247 0.00856992 2024-09-13T14:27:13.799 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (17s) 11s ago 7m 16.0M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (7m) 11s ago 7m 90.7M - 9.4.7 954c08fa6188 2c94b8c7455f 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (6m) 11s ago 6m 19.6M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (6m) 11s ago 6m 17.5M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (112s) 11s ago 11m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (2m) 11s ago 13m 513M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (13m) 11s ago 13m 54.7M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (11m) 11s ago 11m 46.6M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (11m) 11s ago 11m 47.4M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (86s) 11s ago 7m 8530k - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:27:13.800 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (77s) 11s ago 6m 8246k - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (11m) 11s ago 11m 336M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (10m) 11s ago 10m 249M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (10m) 11s ago 10m 338M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (9m) 11s ago 9m 293M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (9m) 11s ago 9m 343M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (9m) 11s ago 9m 377M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (8m) 11s ago 8m 221M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (8m) 11s ago 8m 278M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:27:13.801 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (48s) 11s ago 7m 37.5M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:27:13.801 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:13 smithi042 ceph-mon[40707]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.802 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:13 smithi042 ceph-mon[29349]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:13.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:13 smithi159 ceph-mon[33330]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 86802 86786 115.395 60 0.0298533 0.00866064 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:27:14.486 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:27:14.487 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: Upgrade: Updating grafana.a 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: Deploying daemon grafana.a on smithi159 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: pgmap v78: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/282701829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: Upgrade: Updating grafana.a 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: Deploying daemon grafana.a on smithi159 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: pgmap v78: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:14 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/282701829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:14.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: Upgrade: Updating grafana.a 2024-09-13T14:27:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: Deploying daemon grafana.a on smithi159 2024-09-13T14:27:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: from='client.24877 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: pgmap v78: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:27:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:14 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/282701829' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/2 daemons upgraded", 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T14:27:15.075 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:27:15.076 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:27:15.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 87920 87904 114.446 69.875 0.00990946 0.00873269 2024-09-13T14:27:16.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 89111 89095 113.63 74.4375 0.0072083 0.00879552 2024-09-13T14:27:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:16 smithi042 ceph-mon[40707]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:16 smithi042 ceph-mon[40707]: pgmap v79: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:27:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:16 smithi042 ceph-mon[29349]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:16 smithi042 ceph-mon[29349]: pgmap v79: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:27:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:16 smithi159 ceph-mon[33330]: from='client.15150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:16 smithi159 ceph-mon[33330]: pgmap v79: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:27:17.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 90218 90202 112.741 69.1875 0.0155871 0.00886459 2024-09-13T14:27:18.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 91322 91306 111.883 69 0.00910753 0.00893256 2024-09-13T14:27:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:18 smithi042 ceph-mon[40707]: pgmap v80: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:18.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:18 smithi042 ceph-mon[29349]: pgmap v80: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:18 smithi159 ceph-mon[33330]: pgmap v80: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:19.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 92443 92427 111.078 70.0625 0.00672611 0.00899764 2024-09-13T14:27:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:19] "GET /metrics HTTP/1.1" 200 36223 "" "Prometheus/2.51.0" 2024-09-13T14:27:20.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 93639 93623 110.393 74.75 0.00791593 0.00905279 2024-09-13T14:27:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:20 smithi159 ceph-mon[33330]: pgmap v81: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:20 smithi042 ceph-mon[29349]: pgmap v81: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:20 smithi042 ceph-mon[40707]: pgmap v81: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 999 op/s 2024-09-13T14:27:21.124 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 94807 94791 109.7 73 0.00709099 0.00911076 2024-09-13T14:27:22.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 95956 95940 109.011 71.8125 0.0059675 0.00916787 2024-09-13T14:27:22.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:22 smithi159 ceph-mon[33330]: pgmap v82: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:27:23.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:22 smithi042 ceph-mon[29349]: pgmap v82: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:27:23.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:22 smithi042 ceph-mon[40707]: pgmap v82: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:27:23.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 97091 97075 108.331 70.9375 0.00747214 0.00922625 2024-09-13T14:27:24.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 98240 98224 107.691 71.8125 0.00746471 0.00928074 2024-09-13T14:27:24.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:24 smithi159 ceph-mon[33330]: pgmap v83: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T14:27:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:24 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:24 smithi042 ceph-mon[40707]: pgmap v83: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T14:27:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:24 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:24 smithi042 ceph-mon[29349]: pgmap v83: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2024-09-13T14:27:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:24 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:25.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 99393 99377 107.076 72.0625 0.0177809 0.00933324 2024-09-13T14:27:26.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 100557 100541 106.494 72.75 0.00548257 0.00938498 2024-09-13T14:27:26.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:26 smithi159 ceph-mon[33330]: pgmap v84: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:26 smithi042 ceph-mon[40707]: pgmap v84: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:26 smithi042 ceph-mon[29349]: pgmap v84: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:27.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:27:27.125510+0000 min lat: 0.00114625 max lat: 0.127575 avg lat: 0.0094362 2024-09-13T14:27:27.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:27:27.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 101703 101687 105.913 71.625 0.0174042 0.0094362 2024-09-13T14:27:28.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 102807 102791 105.308 69 0.0190775 0.00949053 2024-09-13T14:27:28.998 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:28 smithi159 systemd[1]: Stopping Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:27:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:28 smithi159 ceph-mon[33330]: pgmap v85: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:29.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:28 smithi042 ceph-mon[29349]: pgmap v85: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:29.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:28 smithi042 ceph-mon[40707]: pgmap v85: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:29.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 103881 103865 104.692 67.125 0.00810378 0.00954648 2024-09-13T14:27:29.494 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:28 smithi159 podman[93068]: 2024-09-13 14:27:28.998047576 +0000 UTC m=+0.252350582 container died 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.expose-services=, version=8.5, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, distribution-scope=public, com.redhat.component=ubi8-container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , release=239.1651231664, vendor=Red Hat, Inc.) 2024-09-13T14:27:29.748 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:29 smithi159 podman[93068]: 2024-09-13 14:27:29.592086833 +0000 UTC m=+0.846389842 container cleanup 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, release=239.1651231664, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, architecture=x86_64, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.buildah.version=1.32.0) 2024-09-13T14:27:29.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:29 smithi159 bash[93068]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a 2024-09-13T14:27:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:29] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:27:30.125 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 104931 104915 104.072 65.625 0.00506297 0.00960332 2024-09-13T14:27:30.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:30 smithi159 ceph-mon[33330]: pgmap v86: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:30.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 podman[93083]: 2024-09-13 14:27:30.386336054 +0000 UTC m=+1.379310085 container remove 2c94b8c7455fc0665fadf15403ae6c8ad637d09a85a6659622f0a80669ce5a0d (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., name=ubi8, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8) 2024-09-13T14:27:30.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T14:27:30.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T14:27:30.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 systemd[1]: Stopped Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:27:30.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service: Consumed 9.752s CPU time. 2024-09-13T14:27:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:30 smithi042 ceph-mon[29349]: pgmap v86: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:30 smithi042 ceph-mon[40707]: pgmap v86: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s 2024-09-13T14:27:31.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 106024 106008 103.513 68.3125 0.0123114 0.00965508 2024-09-13T14:27:31.199 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:30 smithi159 systemd[1]: Starting Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:27:31.498 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 podman[93200]: 2024-09-13 14:27:31.103696391 +0000 UTC m=+0.027685063 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-13T14:27:31.499 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 podman[93200]: 2024-09-13 14:27:31.238129908 +0000 UTC m=+0.162118569 container create 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:27:31.774 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 podman[93200]: 2024-09-13 14:27:31.516500944 +0000 UTC m=+0.440489612 container init 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:27:31.774 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 podman[93200]: 2024-09-13 14:27:31.523703856 +0000 UTC m=+0.447692517 container start 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 bash[93200]: 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 systemd[1]: Started Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749168947Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-13T14:27:31Z 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749688953Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749735648Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749765467Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749790567Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749814025Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749839842Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749864041Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-13T14:27:31.775 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.74988901Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749916446Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749947514Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.749976288Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750002334Z level=info msg=Target target=[all] 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750037344Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750063513Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750088711Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750113546Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750138648Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=settings t=2024-09-13T14:27:31.750164475Z level=info msg="App mode production" 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore t=2024-09-13T14:27:31.750615031Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore t=2024-09-13T14:27:31.750668531Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T14:27:31.776 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.751421493Z level=info msg="Starting DB migrations" 2024-09-13T14:27:31.777 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.773247378Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.78610481Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=12.85358ms 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.786934055Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.787291872Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=358.195µs 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.788144399Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.789674691Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=1.517894ms 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.794048783Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-13T14:27:32.028 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.803435045Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=9.382334ms 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.804492097Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.813662404Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=9.168548ms 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.814663902Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.823606609Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=8.937586ms 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.82438564Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.82468753Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=302.142µs 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.82572813Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.827040683Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=1.314979ms 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.832804126Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-13T14:27:32.029 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.834298754Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=1.493556ms 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.836622204Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.848037226Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=11.41032ms 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.849118381Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.858195243Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=9.069585ms 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.860186213Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.86035938Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=173.762µs 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.861163878Z level=info msg="Executing migration" id="permission kind migration" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.871328926Z level=info msg="Migration successfully executed" id="permission kind migration" duration=10.160088ms 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.872145192Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.882222522Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=10.072626ms 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.885707071Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-13T14:27:32.030 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.896465485Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=10.75909ms 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.897344878Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.899027079Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=1.681291ms 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.900570032Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.901965104Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=1.39561ms 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.902649333Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.90409062Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=1.440968ms 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.904790249Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.904934125Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=144.04µs 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.905601363Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.906899838Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=1.297682ms 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.90840712Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-13T14:27:32.031 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.909573944Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=1.168222ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.912664838Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.937797885Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=25.131279ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.939047453Z level=info msg="Executing migration" id="create correlation v2" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.940363431Z level=info msg="Migration successfully executed" id="create correlation v2" duration=1.311658ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.941050866Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.942379673Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=1.325139ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.942980782Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.94422603Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=1.244765ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.944944928Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.946358959Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=1.413487ms 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.947052668Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-13T14:27:32.032 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.947425846Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=373.419µs 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.948244491Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.949460578Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=1.215509ms 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.950389633Z level=info msg="Executing migration" id="add provisioning column" 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.959817882Z level=info msg="Migration successfully executed" id="add provisioning column" duration=9.425402ms 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.961969074Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.962334609Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=365.048µs 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.967367909Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.967706415Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=338.234µs 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.968652086Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.978352228Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=9.696088ms 2024-09-13T14:27:32.033 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.979193675Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.979741452Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=548.568µs 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.980489076Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.980790642Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=301.735µs 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.982373157Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.983952076Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=1.575226ms 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.984527326Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.985796316Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=1.266795ms 2024-09-13T14:27:32.034 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.986968316Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.988376681Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=1.407952ms 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.989700064Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.990355284Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=632.82µs 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.994670049Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.995083714Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=415.603µs 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.995608868Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.996707856Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=1.09853ms 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.99802247Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:31.999328075Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=1.305447ms 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.000749471Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-13T14:27:32.035 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.001716061Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=967.702µs 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.003468425Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.004913479Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=1.444583ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.006192115Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.007542132Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=1.349786ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.008918319Z level=info msg="Executing migration" id="create anon_device table" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.010060146Z level=info msg="Migration successfully executed" id="create anon_device table" duration=1.14141ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.011775893Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.01307355Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=1.297777ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.014430002Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.015770053Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=1.339897ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.016672191Z level=info msg="Executing migration" id="create signing_key table" 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.017793807Z level=info msg="Migration successfully executed" id="create signing_key table" duration=1.122307ms 2024-09-13T14:27:32.036 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.018477036Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.019699694Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=1.221307ms 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.02117036Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.022541785Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=1.371684ms 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.023148573Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.023881916Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=734.627µs 2024-09-13T14:27:32.037 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.024499712Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-13T14:27:32.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 107062 107046 102.918 64.875 0.0119386 0.00971088 2024-09-13T14:27:32.296 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.033801846Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=9.292463ms 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.035005735Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.037988335Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=2.984639ms 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.039111572Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.040631925Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=1.519153ms 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.041261112Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.042499253Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=1.238169ms 2024-09-13T14:27:32.297 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.043258839Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.044471732Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=1.217089ms 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.044983756Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.046567239Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=1.582653ms 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.04729231Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.048649276Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=1.356454ms 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.051386718Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.052936082Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=1.550548ms 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.053476847Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.054501288Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=1.025547ms 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.055036794Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.055727147Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=690.315µs 2024-09-13T14:27:32.298 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.05673877Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.056873442Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=135.271µs 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.059448451Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.06938112Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=9.930136ms 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.070287476Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.080009999Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=9.718856ms 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.080776955Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.08125438Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=477.858µs 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=migrator t=2024-09-13T14:27:32.082119993Z level=info msg="migrations completed" performed=65 skipped=482 duration=308.990288ms 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore t=2024-09-13T14:27:32.083351779Z level=info msg="Created default organization" 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=secrets t=2024-09-13T14:27:32.087072317Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T14:27:32.299 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugin.store t=2024-09-13T14:27:32.112419811Z level=info msg="Loading plugins..." 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=local.finder t=2024-09-13T14:27:32.19571511Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugin.store t=2024-09-13T14:27:32.195815548Z level=info msg="Plugins loaded" count=55 duration=83.395174ms 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=query_data t=2024-09-13T14:27:32.203135454Z level=info msg="Query Service initialization" 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=live.push_http t=2024-09-13T14:27:32.207316059Z level=info msg="Live Push Gateway initialization" 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.migration t=2024-09-13T14:27:32.209006399Z level=info msg=Starting 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.state.manager t=2024-09-13T14:27:32.218305265Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=infra.usagestats.collector t=2024-09-13T14:27:32.220862745Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.datasources t=2024-09-13T14:27:32.227957768Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.datasources t=2024-09-13T14:27:32.228578178Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.alerting t=2024-09-13T14:27:32.290263355Z level=info msg="starting to provision alerting" 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.alerting t=2024-09-13T14:27:32.290351822Z level=info msg="finished to provision alerting" 2024-09-13T14:27:32.300 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.dashboard t=2024-09-13T14:27:32.291505333Z level=info msg="starting to provision dashboards" 2024-09-13T14:27:32.301 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.state.manager t=2024-09-13T14:27:32.29283649Z level=info msg="Warming state cache for startup" 2024-09-13T14:27:32.301 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.multiorg.alertmanager t=2024-09-13T14:27:32.293619193Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-13T14:27:32.681 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=http.server t=2024-09-13T14:27:32.299111769Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-13T14:27:32.681 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=http.server t=2024-09-13T14:27:32.299754809Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T14:27:32.681 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafanaStorageLogger t=2024-09-13T14:27:32.306112506Z level=info msg="Storage starting" 2024-09-13T14:27:32.681 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.state.manager t=2024-09-13T14:27:32.363417516Z level=info msg="State cache has been initialized" states=0 duration=70.579739ms 2024-09-13T14:27:32.681 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ngalert.scheduler t=2024-09-13T14:27:32.363775686Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ticker t=2024-09-13T14:27:32.363875007Z level=info msg=starting first_tick=2024-09-13T14:27:40Z 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugins.update.checker t=2024-09-13T14:27:32.397829847Z level=info msg="Update check succeeded" duration=104.163175ms 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana.update.checker t=2024-09-13T14:27:32.402654351Z level=info msg="Update check succeeded" duration=110.025538ms 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore.transactions t=2024-09-13T14:27:32.406254104Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore.transactions t=2024-09-13T14:27:32.468047328Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:27:32.682 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=provisioning.dashboard t=2024-09-13T14:27:32.680644552Z level=info msg="finished to provision dashboards" 2024-09-13T14:27:32.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-mon[33330]: pgmap v87: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:32.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:32.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:32.941 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[40707]: pgmap v87: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[29349]: pgmap v87: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:33.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:32 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:33.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 108078 108062 102.321 63.5 0.0112043 0.00976776 2024-09-13T14:27:33.248 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana-apiserver t=2024-09-13T14:27:32.93807571Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-13T14:27:33.249 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:27:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana-apiserver t=2024-09-13T14:27:32.938747513Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-13T14:27:34.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 109115 109099 101.761 64.8125 0.0141477 0.0098214 2024-09-13T14:27:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:34 smithi159 ceph-mon[33330]: pgmap v88: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2024-09-13T14:27:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:34 smithi042 ceph-mon[29349]: pgmap v88: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2024-09-13T14:27:35.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:34 smithi042 ceph-mon[40707]: pgmap v88: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s 2024-09-13T14:27:35.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 110092 110076 101.163 61.0625 0.0545035 0.00987596 2024-09-13T14:27:36.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 111146 111130 100.651 65.875 0.0096681 0.00993021 2024-09-13T14:27:36.976 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:36 smithi159 ceph-mon[33330]: pgmap v89: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:27:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:36 smithi042 ceph-mon[40707]: pgmap v89: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:27:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:36 smithi042 ceph-mon[29349]: pgmap v89: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:27:37.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 112196 112180 100.151 65.625 0.00801324 0.00997956 2024-09-13T14:27:38.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 113223 113207 99.644 64.1875 0.0263591 0.0100303 2024-09-13T14:27:39.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:38 smithi159 ceph-mon[33330]: pgmap v90: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:39.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:38 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:38 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[40707]: pgmap v90: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[29349]: pgmap v90: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:38 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 114230 114214 99.1341 62.9375 0.0092175 0.0100821 2024-09-13T14:27:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:39 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:39 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:39 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:39] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.51.0" 2024-09-13T14:27:40.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 115259 115243 98.657 64.3125 0.0106384 0.0101309 2024-09-13T14:27:40.729 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:40 smithi159 ceph-mon[33330]: pgmap v91: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:40.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:40 smithi042 ceph-mon[40707]: pgmap v91: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:40.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:40 smithi042 ceph-mon[29349]: pgmap v91: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2024-09-13T14:27:41.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 15 116265 116250 98.1742 62.9375 0.010554 0.0101804 2024-09-13T14:27:42.127 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 117285 117269 97.7143 63.6875 0.0114816 0.0102287 2024-09-13T14:27:42.824 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:42 smithi159 ceph-mon[33330]: pgmap v92: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:27:43.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:42 smithi042 ceph-mon[40707]: pgmap v92: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:27:43.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:42 smithi042 ceph-mon[29349]: pgmap v92: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:27:43.126 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 118273 118257 97.241 61.75 0.0110642 0.0102781 2024-09-13T14:27:43.886 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: Upgrade: Finalizing container_image settings 2024-09-13T14:27:43.888 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: pgmap v93: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:27:43.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T14:27:43.890 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:27:44.127 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 119180 119164 96.7143 56.6875 0.0184017 0.0103348 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: Upgrade: Complete! 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:43 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: Upgrade: Finalizing container_image settings 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:27:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.352 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: Upgrade: Finalizing container_image settings 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T14:27:44.353 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: pgmap v93: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T14:27:44.354 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:27:44.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: pgmap v93: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: Upgrade: Complete! 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: Upgrade: Complete! 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:27:44.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:44.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:43 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:45.127 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 120163 120147 96.262 61.4375 0.0165007 0.0103831 2024-09-13T14:27:45.670 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:45 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:45.670 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:45 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:45.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:45 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:46.127 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 121091 121075 95.7776 58 0.0223266 0.0104355 2024-09-13T14:27:46.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-13T14:27:46.666 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:46 smithi042 ceph-mon[40707]: pgmap v94: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:27:46.666 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:46 smithi042 ceph-mon[40707]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:46.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:46 smithi042 ceph-mon[29349]: pgmap v94: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:27:46.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:46 smithi042 ceph-mon[29349]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:46.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:46 smithi159 ceph-mon[33330]: pgmap v94: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T14:27:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:46 smithi159 ceph-mon[33330]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:47.015 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:47.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:27:47.128866+0000 min lat: 0.00114625 max lat: 0.127575 avg lat: 0.0104886 2024-09-13T14:27:47.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:27:47.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 122001 121985 95.2894 56.875 0.0132374 0.0104886 2024-09-13T14:27:48.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 122938 122922 94.8359 58.5625 0.0194074 0.0105391 2024-09-13T14:27:48.737 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:48 smithi042 ceph-mon[40707]: pgmap v95: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:48.737 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:48 smithi042 ceph-mon[29349]: pgmap v95: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:48.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:48 smithi159 ceph-mon[33330]: pgmap v95: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:49.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 123880 123864 94.3973 58.875 0.00934593 0.0105882 2024-09-13T14:27:49.662 INFO:teuthology.orchestra.run.smithi042.stdout:wait for minority of mons to upgrade 2024-09-13T14:27:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:49] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.51.0" 2024-09-13T14:27:50.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 124839 124823 93.9821 59.9375 0.0132653 0.0106351 2024-09-13T14:27:50.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:50 smithi159 ceph-mon[33330]: pgmap v96: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:50 smithi042 ceph-mon[40707]: pgmap v96: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:50.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:50 smithi042 ceph-mon[29349]: pgmap v96: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s 2024-09-13T14:27:51.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 15 125762 125747 93.5507 57.75 0.0165549 0.0106838 2024-09-13T14:27:51.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-13T14:27:51.947 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:52.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 126619 126603 93.0795 53.5 0.0105468 0.010738 2024-09-13T14:27:52.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:52 smithi159 ceph-mon[33330]: pgmap v97: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:27:52.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:52 smithi042 ceph-mon[29349]: pgmap v97: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:27:52.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:52 smithi042 ceph-mon[40707]: pgmap v97: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:27:53.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 127545 127529 92.6701 57.875 0.019324 0.0107852 2024-09-13T14:27:54.129 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 128447 128431 92.2529 56.375 0.021945 0.010834 2024-09-13T14:27:54.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:54 smithi159 ceph-mon[33330]: pgmap v98: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-09-13T14:27:54.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:54 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:54.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:54 smithi042 ceph-mon[29349]: pgmap v98: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-09-13T14:27:54.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:54 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:54.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:54 smithi042 ceph-mon[40707]: pgmap v98: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 50 MiB/s wr, 804 op/s 2024-09-13T14:27:54.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:54 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:27:55.130 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 129283 129267 91.7971 52.25 0.00770164 0.0108885 2024-09-13T14:27:55.733 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:55 smithi042 ceph-mon[29349]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:55.734 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:55 smithi042 ceph-mon[40707]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:55.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:55 smithi159 ceph-mon[33330]: from='client.24895 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:27:56.131 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 130214 130198 91.4194 58.1875 0.00809283 0.0109336 2024-09-13T14:27:56.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:56 smithi159 ceph-mon[33330]: pgmap v99: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:27:56.807 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:56 smithi042 ceph-mon[29349]: pgmap v99: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:27:56.807 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:56 smithi042 ceph-mon[40707]: pgmap v99: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-13T14:27:57.132 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0082 2024-09-13T14:27:57.132 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 131112 2024-09-13T14:27:57.132 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:27:57.132 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:27:57.132 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 91.0417 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 42.5389 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 252.688 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 52.25 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1456 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 680.623 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 4043 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 836 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0109794 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.00727408 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 0.127575 2024-09-13T14:27:57.133 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00114625 2024-09-13T14:27:57.137 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:27:57.656 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:58.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:27:58.787 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:27:58.923 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: pgmap v100: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:27:58.923 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:58.923 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:58.923 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: pgmap v100: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:58.924 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:58 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: pgmap v100: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T14:27:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:58 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:27:59.835 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:27:59 smithi042 ceph-mon[40707]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:59.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:27:59 smithi042 ceph-mon[29349]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:27:59.836 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:27:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:27:59] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.51.0" 2024-09-13T14:27:59.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:27:59 smithi159 ceph-mon[33330]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:28:00.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:00 smithi159 ceph-mon[33330]: pgmap v101: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:28:01.091 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:00 smithi042 ceph-mon[29349]: pgmap v101: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:28:01.092 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:00 smithi042 ceph-mon[40707]: pgmap v101: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 769 op/s 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:02.436 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: Upgrade: Setting container_image for all mgr 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[29349]: pgmap v102: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.14k op/s 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:02.437 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: Upgrade: Setting container_image for all mgr 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:28:02.438 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:02 smithi042 ceph-mon[40707]: pgmap v102: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.14k op/s 2024-09-13T14:28:02.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all mgr 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: Upgrade: It appears safe to stop mon.a 2024-09-13T14:28:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:02 smithi159 ceph-mon[33330]: pgmap v102: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 61 MiB/s wr, 1.14k op/s 2024-09-13T14:28:03.946 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:28:04.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:04 smithi042 ceph-mon[40707]: pgmap v103: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 747 op/s 2024-09-13T14:28:04.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:04 smithi042 ceph-mon[29349]: pgmap v103: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 747 op/s 2024-09-13T14:28:04.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:04 smithi159 ceph-mon[33330]: pgmap v103: 89 pgs: 89 active+clean; 7.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 747 op/s 2024-09-13T14:28:05.068 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:28:05.068 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (69s) 62s ago 8m 16.0M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:28:05.068 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (33s) 27s ago 8m 43.2M - c8b91775d855 3a8b5443f86c 2024-09-13T14:28:05.068 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (7m) 62s ago 7m 19.6M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (7m) 27s ago 7m 17.6M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (2m) 27s ago 12m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (3m) 62s ago 13m 513M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (14m) 62s ago 14m 54.7M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e 86a455d369bb 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (12m) 27s ago 12m 47.3M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (12m) 62s ago 12m 47.4M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (2m) 62s ago 7m 8530k - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (2m) 27s ago 7m 8354k - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (12m) 62s ago 12m 336M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (11m) 62s ago 11m 249M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (11m) 62s ago 11m 338M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (10m) 62s ago 10m 293M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:28:05.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (10m) 27s ago 10m 384M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:28:05.070 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (10m) 27s ago 10m 422M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:28:05.070 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (9m) 27s ago 9m 270M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:28:05.070 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (9m) 27s ago 9m 338M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:28:05.070 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (99s) 27s ago 8m 37.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:28:05.750 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:28:05.750 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 3 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:05.751 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:28:05.752 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 13, 2024-09-13T14:28:05.753 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:05.753 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:28:05.753 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: Upgrade: Updating mon.a 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:28:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[29349]: from='client.? 172.21.15.42:0/273840019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: Upgrade: Updating mon.a 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:05 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/273840019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:06.106 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='client.24898 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: Upgrade: Updating mon.a 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T14:28:06.107 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:05 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/273840019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:06.329 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:28:06.329 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:28:06.329 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:28:06.329 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-13T14:28:06.329 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:28:06.330 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "0/3 daemons upgraded", 2024-09-13T14:28:06.330 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:28:06.330 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:28:06.330 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:28:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[40707]: Deploying daemon mon.a on smithi042 2024-09-13T14:28:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[40707]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[40707]: pgmap v104: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.11k op/s 2024-09-13T14:28:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[29349]: Deploying daemon mon.a on smithi042 2024-09-13T14:28:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[29349]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:06 smithi042 ceph-mon[29349]: pgmap v104: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.11k op/s 2024-09-13T14:28:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:06 smithi159 ceph-mon[33330]: Deploying daemon mon.a on smithi042 2024-09-13T14:28:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:06 smithi159 ceph-mon[33330]: from='client.15180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:06 smithi159 ceph-mon[33330]: pgmap v104: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 36 MiB/s wr, 1.11k op/s 2024-09-13T14:28:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:07 smithi042 systemd[1]: Stopping Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:28:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[29326]: 2024-09-13T14:28:07.645+0000 7fe4e5ae0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:28:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[29326]: 2024-09-13T14:28:07.645+0000 7fe4e5ae0640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T14:28:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:07 smithi042 podman[108388]: 2024-09-13 14:28:07.737056468 +0000 UTC m=+0.247842329 container died 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE Bus STOPPING 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE Bus STOPPED 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE Bus STARTING 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE Serving on http://:::9283 2024-09-13T14:28:08.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:07 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:07] ENGINE Bus STARTED 2024-09-13T14:28:08.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 podman[108388]: 2024-09-13 14:28:08.339906247 +0000 UTC m=+0.850692108 container cleanup 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-13T14:28:08.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 bash[108388]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a 2024-09-13T14:28:08.941 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 podman[108403]: 2024-09-13 14:28:08.65899957 +0000 UTC m=+0.912842585 container remove 86a455d369bb4a4bc25aa7d9c4f9d1e02657472296b0609fa4217f40da64cf51 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=reef-2c01458, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T14:28:09.298 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T14:28:09.298 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 systemd[1]: Stopped Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:28:09.298 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:08 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service: Consumed 17.865s CPU time. 2024-09-13T14:28:09.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:09 smithi042 systemd[1]: Starting Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:28:10.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:09 smithi042 podman[108529]: 2024-09-13 14:28:09.571388164 +0000 UTC m=+0.036847678 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:28:10.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:09 smithi042 podman[108529]: 2024-09-13 14:28:09.712034321 +0000 UTC m=+0.177493823 container create 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True) 2024-09-13T14:28:10.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 podman[108529]: 2024-09-13 14:28:10.005776393 +0000 UTC m=+0.471235894 container init 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T14:28:10.007 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:28:09] "GET /metrics HTTP/1.1" 200 36224 "" "Prometheus/2.51.0" 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 podman[108529]: 2024-09-13 14:28:10.016697794 +0000 UTC m=+0.482157295 container start 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: pidfile_write: ignore empty --pid-file 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: load: jerasure load: lrc 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Git sha 0 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:28:10.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: DB SUMMARY 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: DB Session ID: BF94VDIXYAI20RZSGOVA 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1518 Bytes 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 2805701 ; 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:28:10.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.env: 0x5653fda8f160 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.info_log: 0x5653ffa3b880 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.statistics: (nil) 2024-09-13T14:28:10.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.use_fsync: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:28:10.357 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.db_log_dir: 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.wal_dir: 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:28:10.358 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.write_buffer_manager: 0x5653ffa3e960 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.unordered_write: 0 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:28:10.359 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.row_cache: None 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.wal_filter: None 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.wal_compression: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:28:10.360 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:28:10.361 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_open_files: -1 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:28:10.362 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Compression algorithms supported: 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kZSTD supported: 0 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:28:10.363 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.merge_operator: 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_filter: None 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5653ffa3b4e0) 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:28:10.364 INFO:journalctl@ceph.mon.a.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_type: 0 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_index_type: 0 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_shortening: 1 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: checksum: 4 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: no_block_cache: 0 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache: 0x5653ffa2e850 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_options: 2024-09-13T14:28:10.365 INFO:journalctl@ceph.mon.a.smithi042.stdout: capacity : 536870912 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: num_shard_bits : 4 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: persistent_cache: (nil) 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size: 4096 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_size_deviation: 10 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_restart_interval: 16 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: metadata_block_size: 4096 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: partition_filters: 0 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: use_delta_encoding: 1 2024-09-13T14:28:10.366 INFO:journalctl@ceph.mon.a.smithi042.stdout: filter_policy: bloomfilter 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: whole_key_filtering: 1 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: verify_compression: 0 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: format_version: 5 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_index_compression: 1 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: block_align: 0 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: max_auto_readahead_size: 262144 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: prepopulate_block_cache: 0 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:28:10.367 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression: NoCompression 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.num_levels: 7 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:28:10.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:28:10.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:28:10.370 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:28:10.371 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:28:10.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.ttl: 2592000 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:28:10.373 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12183, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-09-13T14:28:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e71d50f8-8d31-418b-b47a-35930c58253a 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237690062525, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237690078969, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 2527409, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12184, "largest_seqno": 14583, "table_properties": {"data_size": 2517073, "index_size": 6281, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3013, "raw_key_size": 26756, "raw_average_key_size": 22, "raw_value_size": 2493321, "raw_average_value_size": 2072, "num_data_blocks": 285, "num_entries": 1203, "num_filter_entries": 1203, "num_deletions": 259, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237690, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e71d50f8-8d31-418b-b47a-35930c58253a", "db_session_id": "BF94VDIXYAI20RZSGOVA", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237690079114, "job": 1, "event": "recovery_finished"} 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5653ffa5ee00 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: DB pointer 0x5653ffb5e000 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: starting mon.a rank 0 at public addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] at bind addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:28:10.375 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** DB Stats ** 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: L0 1/0 2.41 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:28:10.376 INFO:journalctl@ceph.mon.a.smithi042.stdout: Sum 2/0 10.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 341.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 341.9 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-13T14:28:10.377 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: Cumulative compaction: 0.00 GB write, 111.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: Interval compaction: 0.00 GB write, 111.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: Block cache BinnedLRUCache@0x5653ffa2e850#7 capacity: 512.00 MB usage: 58.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: Block cache entry stats(count,size,portion): DataBlock(3,24.23 KB,0.00462234%) FilterBlock(2,12.06 KB,0.00230074%) IndexBlock(2,22.66 KB,0.00432134%) Misc(1,0.00 KB,0%) 2024-09-13T14:28:10.378 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???) e3 preinit fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).mds e9 new map 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).mds e9 print_map 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: e9 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:28:10.379 INFO:journalctl@ceph.mon.a.smithi042.stdout: legacy client fscid: 1 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: Filesystem 'foo' (1) 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: fs_name foo 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: epoch 8 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: created 2024-09-13T14:20:45.772517+0000 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: modified 2024-09-13T14:20:58.813521+0000 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: tableserver 0 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: root 0 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: session_timeout 60 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: session_autoclose 300 2024-09-13T14:28:10.380 INFO:journalctl@ceph.mon.a.smithi042.stdout: max_file_size 1099511627776 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: max_xattr_size 65536 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: required_client_features {} 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: last_failure 0 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: last_failure_osd_epoch 0 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: max_mds 1 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: in 0 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: up {0=14652} 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: failed 2024-09-13T14:28:10.381 INFO:journalctl@ceph.mon.a.smithi042.stdout: damaged 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: stopped 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: data_pools [4] 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: metadata_pool 3 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: inline_data disabled 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: balancer 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: bal_rank_mask -1 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: standby_count_wanted 1 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: [mds.foo.smithi042.pkkbnw{0:14652} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: Standby daemons: 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: 2024-09-13T14:28:10.382 INFO:journalctl@ceph.mon.a.smithi042.stdout: [mds.foo.smithi159.ypxvst{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).osd e84 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).osd e84 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 bash[108529]: 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 2024-09-13T14:28:10.383 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 systemd[1]: Started Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:28:11.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ignoring --setuser ceph since I am not root 2024-09-13T14:28:11.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ignoring --setgroup ceph since I am not root 2024-09-13T14:28:11.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mgr[89504]: -- 172.21.15.42:0/1329556577 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a6a10190e0 con 0x55a6a0ff6800 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: pgmap v106: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 735 op/s 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a calling monitor election 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: monmap epoch 3 2024-09-13T14:28:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: min_mon_release 18 (reef) 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: election_strategy: 1 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: overall HEALTH_OK 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: from='mgr.24851 ' entity='' 2024-09-13T14:28:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[108570]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: pgmap v106: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 735 op/s 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: mon.a calling monitor election 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: monmap epoch 3 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:11.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: min_mon_release 18 (reef) 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: election_strategy: 1 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: overall HEALTH_OK 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: from='mgr.24851 ' entity='' 2024-09-13T14:28:11.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:10 smithi042 ceph-mon[40707]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-13T14:28:11.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: ignoring --setuser ceph since I am not root 2024-09-13T14:28:11.249 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: ignoring --setgroup ceph since I am not root 2024-09-13T14:28:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: pgmap v106: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 735 op/s 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 172.21.15.42:0/501569478' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: mon.a calling monitor election 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: monmap epoch 3 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:11.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: min_mon_release 18 (reef) 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: election_strategy: 1 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: mgrmap e35: y(active, since 3m), standbys: x 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: from='mgr.24851 ' entity='' 2024-09-13T14:28:11.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:10 smithi159 ceph-mon[33330]: mgrmap e36: y(active, since 3m), standbys: x 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: Standby manager daemon x restarted 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: Standby manager daemon x started 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/crt"} : dispatch 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/crt"} : dispatch 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/key"} : dispatch 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-mon[33330]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/key"} : dispatch 2024-09-13T14:28:18.499 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:28:18] ENGINE Bus STARTING 2024-09-13T14:28:18.500 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: CherryPy Checker: 2024-09-13T14:28:18.500 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: The Application mounted at '' has an empty config. 2024-09-13T14:28:18.500 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: 2024-09-13T14:28:18.500 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:28:18] ENGINE Serving on http://:::9283 2024-09-13T14:28:18.500 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 14:28:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x[80711]: [13/Sep/2024:14:28:18] ENGINE Bus STARTED 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: Standby manager daemon x restarted 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: Standby manager daemon x started 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/crt"} : dispatch 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/crt"} : dispatch 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/key"} : dispatch 2024-09-13T14:28:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[108570]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/key"} : dispatch 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: Standby manager daemon x restarted 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: Standby manager daemon x started 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/crt"} : dispatch 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/crt"} : dispatch 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/x/key"} : dispatch 2024-09-13T14:28:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:18 smithi042 ceph-mon[40707]: from='mgr.? 172.21.15.159:0/1399306515' entity='mgr.x' cmd={"prefix": "config-key get", "key": "mgr/dashboard/key"} : dispatch 2024-09-13T14:28:19.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:19 smithi159 ceph-mon[33330]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-13T14:28:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:19 smithi042 ceph-mon[40707]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-13T14:28:19.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:19 smithi042 ceph-mon[108570]: mgrmap e37: y(active, since 3m), standbys: x 2024-09-13T14:28:20.498 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=infra.usagestats t=2024-09-13T14:28:20.313228458Z level=info msg="Usage stats are ready to report" 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: Active manager daemon y restarted 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: Activating manager daemon y 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: mgrmap e38: y(active, starting, since 0.0175977s), standbys: x 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"} : dispatch 2024-09-13T14:28:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:28:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:20 smithi159 ceph-mon[33330]: Manager daemon y is now available 2024-09-13T14:28:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: Active manager daemon y restarted 2024-09-13T14:28:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: Activating manager daemon y 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: mgrmap e38: y(active, starting, since 0.0175977s), standbys: x 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:28:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:28:20.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[40707]: Manager daemon y is now available 2024-09-13T14:28:20.601 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:28:20.218Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephPoolGrowthWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.42:8443: connect: connection refused" 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: Active manager daemon y restarted 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: Activating manager daemon y 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: mgrmap e38: y(active, starting, since 0.0175977s), standbys: x 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi042.pkkbnw"} : dispatch 2024-09-13T14:28:20.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "foo.smithi159.ypxvst"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "y", "id": "y"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr metadata", "who": "x", "id": "x"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T14:28:20.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-mon[108570]: Manager daemon y is now available 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.874+0000 7f23e3861640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:20] ENGINE Bus STARTING 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: CherryPy Checker: 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: The Application mounted at '' has an empty config. 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:21.132 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.921+0000 7f23b7bfc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.921+0000 7f23b7bfc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.921+0000 7f23b7bfc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.921+0000 7f23b7bfc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.921+0000 7f23b7bfc640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.924+0000 7f23b0bee640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.924+0000 7f23b0bee640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.924+0000 7f23b0bee640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.924+0000 7f23b0bee640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:20.924+0000 7f23b0bee640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:21] ENGINE Serving on http://:::9283 2024-09-13T14:28:21.133 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:14:28:21] ENGINE Bus STARTED 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: mgrmap e39: y(active, since 1.02146s), standbys: x 2024-09-13T14:28:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:21 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:28:21.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:21.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:21.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: mgrmap e39: y(active, since 1.02146s), standbys: x 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: mgrmap e39: y(active, since 1.02146s), standbys: x 2024-09-13T14:28:21.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-13T14:28:22.348 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 14:28:21 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T14:28:21.974Z caller=notify.go:860 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephPoolGrowthWarning\"}" msg="Notify success" attempts=3 duration=307.342381ms 2024-09-13T14:28:22.987 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:22 smithi159 ceph-mon[33330]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:22.987 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:22 smithi159 ceph-mon[33330]: pgmap v3: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:22.988 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:22 smithi159 ceph-mon[33330]: pgmap v4: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:22.988 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:22 smithi159 ceph-mon[33330]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:28:23.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:23.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[108570]: pgmap v3: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:23.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[108570]: pgmap v4: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:23.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[108570]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:28:23.189 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[40707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T14:28:23.189 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[40707]: pgmap v3: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:23.189 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[40707]: pgmap v4: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:23.189 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:22 smithi042 ceph-mon[40707]: mgrmap e40: y(active, since 2s), standbys: x 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:23.272+0000 7f23928f8640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:14:28:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: Traceback (most recent call last): 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._connections.run(self.expiration_interval) 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._run(expiration_interval) 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:28:23.600 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: s = self.context.wrap_socket( 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: return self.sslsocket_class._create( 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self.do_handshake() 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: self._sslobj.do_handshake() 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:28:23.601 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:25.174 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: [13/Sep/2024:14:28:23] ENGINE Bus STARTING 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: [13/Sep/2024:14:28:23] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: [13/Sep/2024:14:28:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: Traceback (most recent call last): 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._run(expiration_interval) 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:28:25.175 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: s = self.context.wrap_socket( 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: return self.sslsocket_class._create( 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: self.do_handshake() 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: self._sslobj.do_handshake() 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: [13/Sep/2024:14:28:23] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: [13/Sep/2024:14:28:23] ENGINE Bus STARTED 2024-09-13T14:28:25.176 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: pgmap v5: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:25.177 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:24 smithi159 ceph-mon[33330]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: [13/Sep/2024:14:28:23] ENGINE Bus STARTING 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: [13/Sep/2024:14:28:23] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: [13/Sep/2024:14:28:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:28:25.316 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: self.do_handshake() 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:28:25.317 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: [13/Sep/2024:14:28:23] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: [13/Sep/2024:14:28:23] ENGINE Bus STARTED 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: pgmap v5: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[40707]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: [13/Sep/2024:14:28:23] ENGINE Bus STARTING 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: [13/Sep/2024:14:28:23] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: [13/Sep/2024:14:28:23] ENGINE Error in HTTPServer.serve 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout: Traceback (most recent call last): 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T14:28:25.318 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._run(expiration_interval) 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: self.do_handshake() 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T14:28:25.319 INFO:journalctl@ceph.mon.a.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T14:28:25.320 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: [13/Sep/2024:14:28:23] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T14:28:25.320 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: [13/Sep/2024:14:28:23] ENGINE Bus STARTED 2024-09-13T14:28:25.320 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: pgmap v5: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:25.320 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:24 smithi042 ceph-mon[108570]: mgrmap e41: y(active, since 4s), standbys: x 2024-09-13T14:28:26.203 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:25 smithi042 ceph-mon[40707]: pgmap v6: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:26.203 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:25 smithi042 ceph-mon[108570]: pgmap v6: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:26.241 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:25 smithi159 ceph-mon[33330]: pgmap v6: 89 pgs: 89 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail 2024-09-13T14:28:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:27 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:27 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:27 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:27 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:28.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.130 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:28 smithi159 ceph-mon[33330]: pgmap v7: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 KiB/s rd, 441 B/s wr, 567 op/s 2024-09-13T14:28:29.130 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.130 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.130 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.130 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.321 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[108570]: pgmap v7: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 KiB/s rd, 441 B/s wr, 567 op/s 2024-09-13T14:28:29.321 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.321 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[40707]: pgmap v7: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 25 KiB/s rd, 441 B/s wr, 567 op/s 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.322 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:29.323 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:28 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:28:29] "GET /metrics HTTP/1.1" 200 33466 "" "Prometheus/2.51.0" 2024-09-13T14:28:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:30 smithi042 ceph-mon[40707]: pgmap v8: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 440 op/s 2024-09-13T14:28:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:30 smithi042 ceph-mon[108570]: pgmap v8: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 440 op/s 2024-09-13T14:28:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:30 smithi159 ceph-mon[33330]: pgmap v8: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 20 KiB/s rd, 342 B/s wr, 440 op/s 2024-09-13T14:28:32.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:32 smithi159 ceph-mon[33330]: pgmap v9: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 280 B/s wr, 738 op/s 2024-09-13T14:28:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:32 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:32 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:33.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[40707]: pgmap v9: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 280 B/s wr, 738 op/s 2024-09-13T14:28:33.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:33.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[108570]: pgmap v9: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 16 KiB/s rd, 280 B/s wr, 738 op/s 2024-09-13T14:28:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:32 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:33 smithi159 ceph-mon[33330]: pgmap v10: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:33 smithi042 ceph-mon[108570]: pgmap v10: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:33 smithi042 ceph-mon[40707]: pgmap v10: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:28:36.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:35 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:28:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[108570]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T14:28:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:35 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.conf 2024-09-13T14:28:36.954 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:28:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: pgmap v11: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:36 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: pgmap v11: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: Updating smithi159:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: Updating smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: pgmap v11: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 674 op/s 2024-09-13T14:28:37.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[108570]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[40707]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:37.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:28:37.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:37 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:28:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:37 smithi159 ceph-mon[33330]: Updating smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T14:28:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:37 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:28:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:37 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:28:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:37 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:28:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:37 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["c"]} : dispatch 2024-09-13T14:28:38.088 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (102s) 11s ago 8m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (66s) 10s ago 8m 75.0M - c8b91775d855 3a8b5443f86c 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (7m) 11s ago 7m 20.7M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (7m) 10s ago 7m 18.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (3m) 10s ago 12m 451M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (3m) 11s ago 14m 495M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (28s) 11s ago 14m 39.3M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (13m) 10s ago 13m 56.3M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (13m) 11s ago 13m 54.9M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e a2f1a939abaa 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (2m) 11s ago 8m 10.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (2m) 10s ago 8m 9.92M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:28:38.089 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (12m) 11s ago 12m 390M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (12m) 11s ago 12m 334M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (11m) 11s ago 11m 474M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (11m) 11s ago 11m 483M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (10m) 10s ago 10m 419M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (10m) 10s ago 10m 455M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (10m) 10s ago 10m 307M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (9m) 10s ago 9m 387M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:28:38.090 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (2m) 10s ago 9m 40.7M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:28:38.775 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:28:38.775 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:28:38.775 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2, 2024-09-13T14:28:38.775 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:28:38.775 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 12, 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:28:38.776 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:28:38.777 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:28:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[40707]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[40707]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:28:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[40707]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[40707]: pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.02k op/s 2024-09-13T14:28:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[108570]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[108570]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:28:39.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[108570]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:38 smithi042 ceph-mon[108570]: pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.02k op/s 2024-09-13T14:28:39.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:38 smithi159 ceph-mon[33330]: from='client.25091 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:38 smithi159 ceph-mon[33330]: Upgrade: It appears safe to stop mon.c 2024-09-13T14:28:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:38 smithi159 ceph-mon[33330]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:38 smithi159 ceph-mon[33330]: pgmap v12: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 15 KiB/s rd, 255 B/s wr, 1.02k op/s 2024-09-13T14:28:39.351 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:28:39.351 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:28:39.351 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:28:39.351 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-13T14:28:39.352 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:28:39.352 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/3 daemons upgraded", 2024-09-13T14:28:39.352 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:28:39.352 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:28:39.352 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:28:40.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='client.24955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='client.? 172.21.15.42:0/2839544785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[40707]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:28:39] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='client.24955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2839544785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:40.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:40.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:28:40.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:28:40.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='client.24955 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/2839544785' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T14:28:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:28:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:28:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:39 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:28:41.092 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[40707]: Upgrade: Updating mon.c 2024-09-13T14:28:41.092 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[40707]: Deploying daemon mon.c on smithi042 2024-09-13T14:28:41.092 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[40707]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:41.092 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[40707]: pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:28:41.093 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[108570]: Upgrade: Updating mon.c 2024-09-13T14:28:41.093 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[108570]: Deploying daemon mon.c on smithi042 2024-09-13T14:28:41.093 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[108570]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:41.093 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:40 smithi042 ceph-mon[108570]: pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:28:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:40 smithi159 ceph-mon[33330]: Upgrade: Updating mon.c 2024-09-13T14:28:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:40 smithi159 ceph-mon[33330]: Deploying daemon mon.c on smithi042 2024-09-13T14:28:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:40 smithi159 ceph-mon[33330]: from='client.25109 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:28:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:40 smithi159 ceph-mon[33330]: pgmap v13: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2024-09-13T14:28:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:41 smithi159 ceph-mon[33330]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:28:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:41 smithi042 systemd[1]: Stopping Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:28:42.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:41 smithi042 ceph-mon[40707]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:28:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:41 smithi042 ceph-mon[108570]: pgmap v14: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:28:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c[40684]: 2024-09-13T14:28:42.429+0000 7f314437a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:28:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c[40684]: 2024-09-13T14:28:42.429+0000 7f314437a640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:28:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:42 smithi042 podman[113766]: 2024-09-13 14:28:42.586178707 +0000 UTC m=+0.333537170 container died a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=reef-2c01458, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T14:28:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 podman[113766]: 2024-09-13 14:28:43.010114037 +0000 UTC m=+0.757472489 container cleanup a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:28:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 bash[113766]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c 2024-09-13T14:28:43.616 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 podman[113781]: 2024-09-13 14:28:43.362073478 +0000 UTC m=+0.763938327 container remove a2f1a939abaac8ef7641cd449363e2a123f0dc18fb23d4499424f08c7fd0decf (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:28:43.950 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T14:28:43.951 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 systemd[1]: Stopped Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:28:43.951 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.c.service: Consumed 10.302s CPU time. 2024-09-13T14:28:44.297 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:43 smithi042 systemd[1]: Starting Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:28:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 podman[113893]: 2024-09-13 14:28:44.204140196 +0000 UTC m=+0.035339009 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:28:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 podman[113893]: 2024-09-13 14:28:44.330472142 +0000 UTC m=+0.161670944 container create 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:28:45.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 podman[113893]: 2024-09-13 14:28:44.621639832 +0000 UTC m=+0.452838639 container init 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-09-13T14:28:45.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 podman[113893]: 2024-09-13 14:28:44.629229515 +0000 UTC m=+0.460428319 container start 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T14:28:45.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: pidfile_write: ignore empty --pid-file 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: load: jerasure load: lrc 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Git sha 0 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: DB SUMMARY 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: DB Session ID: JGYUIOWHGGENRKNM1J5R 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:28:45.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7454122 ; 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:28:45.104 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.env: 0x55dab9c23160 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.info_log: 0x55dabc399880 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.statistics: (nil) 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.use_fsync: 0 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:28:45.105 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.db_log_dir: 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.wal_dir: 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:28:45.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.write_buffer_manager: 0x55dabc39c960 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.unordered_write: 0 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:28:45.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.row_cache: None 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.wal_filter: None 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.wal_compression: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:28:45.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:28:45.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_open_files: -1 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:28:45.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Compression algorithms supported: 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kZSTD supported: 0 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-09-13T14:28:45.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.merge_operator: 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_filter: None 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dabc3994e0) 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:28:45.112 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_type: 0 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: data_block_index_type: 0 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_shortening: 1 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: checksum: 4 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: no_block_cache: 0 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache: 0x55dabc38c850 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_options: 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: capacity : 536870912 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: num_shard_bits : 4 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:28:45.113 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: persistent_cache: (nil) 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_size: 4096 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_size_deviation: 10 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_restart_interval: 16 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: metadata_block_size: 4096 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: partition_filters: 0 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: use_delta_encoding: 1 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: filter_policy: bloomfilter 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: whole_key_filtering: 1 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: verify_compression: 0 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:28:45.114 INFO:journalctl@ceph.mon.c.smithi042.stdout: format_version: 5 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: enable_index_compression: 1 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: block_align: 0 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: max_auto_readahead_size: 262144 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: prepopulate_block_cache: 0 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression: NoCompression 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:28:45.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.num_levels: 7 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:28:45.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:28:45.117 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:28:45.118 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:28:45.118 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:28:45.118 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:28:45.118 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:28:45.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:28:45.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:28:45.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.ttl: 2592000 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:28:45.122 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12784, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 7d0584dd-e794-46ef-9eb4-e5f00afbee91 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237724684037, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237724710338, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4586895, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12789, "largest_seqno": 16026, "table_properties": {"data_size": 4573992, "index_size": 8208, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3653, "raw_key_size": 32627, "raw_average_key_size": 22, "raw_value_size": 4544749, "raw_average_value_size": 3117, "num_data_blocks": 369, "num_entries": 1458, "num_filter_entries": 1458, "num_deletions": 259, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237724, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "7d0584dd-e794-46ef-9eb4-e5f00afbee91", "db_session_id": "JGYUIOWHGGENRKNM1J5R", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:28:45.123 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237724710554, "job": 1, "event": "recovery_finished"} 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55dabc3bce00 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: DB pointer 0x55dabc4bc000 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** DB Stats ** 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:45.124 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: L0 1/0 4.37 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Sum 2/0 12.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:28:45.125 INFO:journalctl@ceph.mon.c.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 436.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 436.4 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:28:45.126 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: Cumulative compaction: 0.00 GB write, 125.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: Interval compaction: 0.00 GB write, 125.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: Block cache BinnedLRUCache@0x55dabc38c850#7 capacity: 512.00 MB usage: 54.45 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: Block cache entry stats(count,size,portion): DataBlock(3,16.23 KB,0.00309646%) FilterBlock(2,13.06 KB,0.00249147%) IndexBlock(2,25.16 KB,0.00479817%) Misc(1,0.00 KB,0%) 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: starting mon.c rank 2 at public addrs [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] at bind addrs [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???) e3 preinit fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).mds e9 new map 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).mds e9 print_map 2024-09-13T14:28:45.127 INFO:journalctl@ceph.mon.c.smithi042.stdout: e9 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: legacy client fscid: 1 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: Filesystem 'foo' (1) 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: fs_name foo 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: epoch 8 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: created 2024-09-13T14:20:45.772517+0000 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: modified 2024-09-13T14:20:58.813521+0000 2024-09-13T14:28:45.128 INFO:journalctl@ceph.mon.c.smithi042.stdout: tableserver 0 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: root 0 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: session_timeout 60 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: session_autoclose 300 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: max_file_size 1099511627776 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: max_xattr_size 65536 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: required_client_features {} 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: last_failure 0 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: last_failure_osd_epoch 0 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: max_mds 1 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: in 0 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: up {0=14652} 2024-09-13T14:28:45.129 INFO:journalctl@ceph.mon.c.smithi042.stdout: failed 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: damaged 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: stopped 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: data_pools [4] 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: metadata_pool 3 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: inline_data disabled 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: balancer 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: bal_rank_mask -1 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: standby_count_wanted 1 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: [mds.foo.smithi042.pkkbnw{0:14652} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: Standby daemons: 2024-09-13T14:28:45.130 INFO:journalctl@ceph.mon.c.smithi042.stdout: 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout: [mds.foo.smithi159.ypxvst{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 ceph-mon[113926]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 bash[113893]: 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 2024-09-13T14:28:45.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:44 smithi042 systemd[1]: Started Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:28:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:28:49] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:28:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:51 smithi042 ceph-mon[113926]: mon.a calling monitor election 2024-09-13T14:28:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:51 smithi042 ceph-mon[113926]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:51 smithi042 ceph-mon[108570]: mon.a calling monitor election 2024-09-13T14:28:51.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:51 smithi042 ceph-mon[108570]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:51.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:51 smithi159 ceph-mon[33330]: mon.a calling monitor election 2024-09-13T14:28:51.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:51 smithi159 ceph-mon[33330]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:28:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: mon.c calling monitor election 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: mon.b calling monitor election 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: monmap epoch 3 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: min_mon_release 18 (reef) 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: election_strategy: 1 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: mgrmap e41: y(active, since 31s), standbys: x 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: overall HEALTH_OK 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[108570]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: mon.c calling monitor election 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: mon.b calling monitor election 2024-09-13T14:28:52.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: monmap epoch 3 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: min_mon_release 18 (reef) 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: election_strategy: 1 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: mgrmap e41: y(active, since 31s), standbys: x 2024-09-13T14:28:52.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: overall HEALTH_OK 2024-09-13T14:28:52.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:52 smithi042 ceph-mon[113926]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: mon.c calling monitor election 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: mon.b calling monitor election 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: monmap epoch 3 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: last_changed 2024-09-13T14:15:33.354690+0000 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: min_mon_release 18 (reef) 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: election_strategy: 1 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:28:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:28:52.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:28:52.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: mgrmap e41: y(active, since 31s), standbys: x 2024-09-13T14:28:52.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:28:52.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:52 smithi159 ceph-mon[33330]: pgmap v19: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:28:54.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:53 smithi042 ceph-mon[108570]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:54.239 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:53 smithi042 ceph-mon[113926]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:53 smithi159 ceph-mon[33330]: pgmap v20: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:55.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:28:54.756+0000 7f23e3861640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T14:28:56.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:56 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:56 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:56 smithi159 ceph-mon[33330]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[108570]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:56 smithi042 ceph-mon[113926]: pgmap v21: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:28:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:57 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:57 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:57.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:57.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:57.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:57.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:28:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:58 smithi042 ceph-mon[113926]: pgmap v22: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:28:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:58 smithi042 ceph-mon[108570]: pgmap v22: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:28:58.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:58 smithi159 ceph-mon[33330]: pgmap v22: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:29:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:28:59 smithi042 ceph-mon[113926]: pgmap v23: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-09-13T14:29:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:28:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:28:59] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-13T14:29:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:28:59 smithi042 ceph-mon[108570]: pgmap v23: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-09-13T14:29:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:28:59 smithi159 ceph-mon[33330]: pgmap v23: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 711 op/s 2024-09-13T14:29:02.205 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:01 smithi042 ceph-mon[113926]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:02.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:01 smithi042 ceph-mon[108570]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:01 smithi159 ceph-mon[33330]: pgmap v24: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:29:03.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all grafana 2024-09-13T14:29:03.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all loki 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all promtail 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Finalizing container_image settings 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:29:03.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: Upgrade: Complete! 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.605 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:29:03.606 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all grafana 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.607 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all loki 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all promtail 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Finalizing container_image settings 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:29:03.608 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.609 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: Upgrade: Complete! 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.610 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T14:29:03.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all grafana 2024-09-13T14:29:03.751 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all loki 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all promtail 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-13T14:29:03.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Finalizing container_image settings 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:29:03.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: Upgrade: Complete! 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:03.754 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:29:03.755 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:04.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:04 smithi042 ceph-mon[108570]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:04.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:04 smithi042 ceph-mon[113926]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:04.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:04 smithi159 ceph-mon[33330]: pgmap v25: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:05 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:05 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:29:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:06 smithi042 ceph-mon[113926]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:06 smithi042 ceph-mon[108570]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:06 smithi159 ceph-mon[33330]: pgmap v26: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-09-13T14:29:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:07 smithi159 ceph-mon[33330]: pgmap v27: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:07 smithi042 ceph-mon[113926]: pgmap v27: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:07 smithi042 ceph-mon[108570]: pgmap v27: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-13T14:29:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:09 smithi042 ceph-mon[108570]: pgmap v28: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:29:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:09] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-09-13T14:29:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:09 smithi042 ceph-mon[113926]: pgmap v28: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:29:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:09 smithi159 ceph-mon[33330]: pgmap v28: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-09-13T14:29:10.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:29:11.021 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:29:11.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:10 smithi159 ceph-mon[33330]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:11.305 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:10 smithi042 ceph-mon[108570]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:11.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:10 smithi042 ceph-mon[113926]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:29:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:11 smithi159 ceph-mon[33330]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:29:12.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:11 smithi042 ceph-mon[113926]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:29:12.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:11 smithi042 ceph-mon[108570]: pgmap v29: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:29:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:13 smithi159 ceph-mon[33330]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:14.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:13 smithi042 ceph-mon[108570]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:14.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:13 smithi042 ceph-mon[113926]: pgmap v30: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:15 smithi159 ceph-mon[33330]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:15 smithi042 ceph-mon[108570]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:15 smithi042 ceph-mon[113926]: pgmap v31: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:17 smithi159 ceph-mon[33330]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:17 smithi042 ceph-mon[108570]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:17 smithi042 ceph-mon[113926]: pgmap v32: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:19 smithi042 ceph-mon[108570]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:29:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:19] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-09-13T14:29:20.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:19 smithi042 ceph-mon[113926]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:29:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:19 smithi159 ceph-mon[33330]: pgmap v33: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2024-09-13T14:29:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:21 smithi159 ceph-mon[33330]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:21 smithi042 ceph-mon[108570]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:21 smithi042 ceph-mon[113926]: pgmap v34: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:23 smithi159 ceph-mon[33330]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:23 smithi042 ceph-mon[113926]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:23 smithi042 ceph-mon[108570]: pgmap v35: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:25 smithi159 ceph-mon[33330]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:25 smithi042 ceph-mon[108570]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:25 smithi042 ceph-mon[113926]: pgmap v36: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-13T14:29:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:27 smithi159 ceph-mon[33330]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:27 smithi042 ceph-mon[108570]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:27 smithi042 ceph-mon[113926]: pgmap v37: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:29:30.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:29 smithi042 ceph-mon[108570]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:30.101 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:29] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-13T14:29:30.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:29 smithi042 ceph-mon[113926]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:29 smithi159 ceph-mon[33330]: pgmap v38: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:31 smithi159 ceph-mon[33330]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:29:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:31 smithi042 ceph-mon[108570]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:29:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:31 smithi042 ceph-mon[113926]: pgmap v39: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T14:29:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:33 smithi159 ceph-mon[33330]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:33 smithi042 ceph-mon[108570]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:33 smithi042 ceph-mon[113926]: pgmap v40: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:36.053 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:35 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:36 smithi042 ceph-mon[113926]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:36 smithi042 ceph-mon[108570]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:36 smithi159 ceph-mon[33330]: pgmap v41: 89 pgs: 89 active+clean; 3.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 743 op/s 2024-09-13T14:29:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:37 smithi159 ceph-mon[33330]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:38.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:37 smithi042 ceph-mon[113926]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:37 smithi042 ceph-mon[108570]: pgmap v42: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:39 smithi042 ceph-mon[108570]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:29:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:39] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-09-13T14:29:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:39 smithi042 ceph-mon[113926]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:29:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:39 smithi159 ceph-mon[33330]: pgmap v43: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:29:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:41 smithi159 ceph-mon[33330]: pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:41 smithi042 ceph-mon[113926]: pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:41 smithi042 ceph-mon[108570]: pgmap v44: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:29:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:43 smithi159 ceph-mon[33330]: pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:43 smithi042 ceph-mon[113926]: pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:43 smithi042 ceph-mon[108570]: pgmap v45: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:45 smithi159 ceph-mon[33330]: pgmap v46: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:45 smithi042 ceph-mon[113926]: pgmap v46: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:45 smithi042 ceph-mon[108570]: pgmap v46: 89 pgs: 89 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:29:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:47 smithi159 ceph-mon[33330]: pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:47 smithi042 ceph-mon[113926]: pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:47 smithi042 ceph-mon[108570]: pgmap v47: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:29:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:49 smithi042 ceph-mon[108570]: pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:29:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:49] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-09-13T14:29:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:49 smithi042 ceph-mon[113926]: pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:29:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:49 smithi159 ceph-mon[33330]: pgmap v48: 89 pgs: 89 active+clean; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-09-13T14:29:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:50 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:51.347 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:51.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:29:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:51 smithi159 ceph-mon[33330]: pgmap v49: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-13T14:29:52.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:51 smithi042 ceph-mon[108570]: pgmap v49: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-13T14:29:52.257 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:51 smithi042 ceph-mon[113926]: pgmap v49: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-09-13T14:29:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:52 smithi159 ceph-mon[33330]: mgrmap e42: y(active, since 92s), standbys: x 2024-09-13T14:29:53.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:52 smithi042 ceph-mon[108570]: mgrmap e42: y(active, since 92s), standbys: x 2024-09-13T14:29:53.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:52 smithi042 ceph-mon[113926]: mgrmap e42: y(active, since 92s), standbys: x 2024-09-13T14:29:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:53 smithi159 ceph-mon[33330]: pgmap v50: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:53 smithi042 ceph-mon[108570]: pgmap v50: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:53 smithi042 ceph-mon[113926]: pgmap v50: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:55 smithi159 ceph-mon[33330]: pgmap v51: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:55 smithi042 ceph-mon[108570]: pgmap v51: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:55 smithi042 ceph-mon[113926]: pgmap v51: 89 pgs: 89 active+clean; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 735 op/s 2024-09-13T14:29:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:57 smithi159 ceph-mon[33330]: pgmap v52: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:29:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:57 smithi042 ceph-mon[113926]: pgmap v52: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:29:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:57 smithi042 ceph-mon[108570]: pgmap v52: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:30:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:29:59 smithi042 ceph-mon[108570]: pgmap v53: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 743 op/s 2024-09-13T14:30:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:29:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:29:59] "GET /metrics HTTP/1.1" 200 36206 "" "Prometheus/2.51.0" 2024-09-13T14:30:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:29:59 smithi042 ceph-mon[113926]: pgmap v53: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 743 op/s 2024-09-13T14:30:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:29:59 smithi159 ceph-mon[33330]: pgmap v53: 89 pgs: 89 active+clean; 1.8 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 255 B/s wr, 743 op/s 2024-09-13T14:30:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:00 smithi159 ceph-mon[33330]: overall HEALTH_OK 2024-09-13T14:30:01.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:00 smithi042 ceph-mon[108570]: overall HEALTH_OK 2024-09-13T14:30:01.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:00 smithi042 ceph-mon[113926]: overall HEALTH_OK 2024-09-13T14:30:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:01 smithi159 ceph-mon[33330]: pgmap v54: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:30:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:01 smithi042 ceph-mon[108570]: pgmap v54: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:30:02.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:01 smithi042 ceph-mon[113926]: pgmap v54: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 255 B/s wr, 1.11k op/s 2024-09-13T14:30:03.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:03.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:03.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:03 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:04 smithi042 ceph-mon[108570]: pgmap v55: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:04 smithi042 ceph-mon[113926]: pgmap v55: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:04.372 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:04 smithi159 ceph-mon[33330]: pgmap v55: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:05 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:07.069 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:06 smithi042 ceph-mon[113926]: pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:07.069 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:06 smithi042 ceph-mon[108570]: pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:06 smithi159 ceph-mon[33330]: pgmap v56: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2024-09-13T14:30:08.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:07 smithi159 ceph-mon[33330]: pgmap v57: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:30:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:07 smithi042 ceph-mon[113926]: pgmap v57: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:30:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:07 smithi042 ceph-mon[108570]: pgmap v57: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-13T14:30:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:09] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-13T14:30:10.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: pgmap v58: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:10 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.539 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.539 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.539 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.539 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: pgmap v58: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: pgmap v58: 89 pgs: 89 active+clean; 1.3 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 718 op/s 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:10.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:10.541 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:11 smithi159 ceph-mon[33330]: pgmap v59: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:30:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:11 smithi042 ceph-mon[113926]: pgmap v59: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:30:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:11 smithi042 ceph-mon[108570]: pgmap v59: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:30:13.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-13T14:30:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:13 smithi042 ceph-mon[108570]: pgmap v60: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:13 smithi042 ceph-mon[113926]: pgmap v60: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:14.210 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:13 smithi159 ceph-mon[33330]: pgmap v60: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:15 smithi159 ceph-mon[33330]: pgmap v61: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:15 smithi042 ceph-mon[108570]: pgmap v61: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:15 smithi042 ceph-mon[113926]: pgmap v61: 89 pgs: 89 active+clean; 1.0 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:30:16.906 INFO:teuthology.orchestra.run.smithi042.stdout:wait for majority of mons to upgrade 2024-09-13T14:30:17.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-13T14:30:17.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:17 smithi042 ceph-mon[108570]: pgmap v62: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:18.052 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:18.229 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:17 smithi042 ceph-mon[113926]: pgmap v62: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:17 smithi159 ceph-mon[33330]: pgmap v62: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:19 smithi042 ceph-mon[108570]: pgmap v63: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:30:20.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:19 smithi042 ceph-mon[113926]: pgmap v63: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:30:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:19] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-13T14:30:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:19 smithi159 ceph-mon[33330]: pgmap v63: 89 pgs: 89 active+clean; 807 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s 2024-09-13T14:30:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:20 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:21.345 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:21.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:21 smithi159 ceph-mon[33330]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:21 smithi159 ceph-mon[33330]: pgmap v64: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:22.272 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:21 smithi042 ceph-mon[108570]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.272 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:21 smithi042 ceph-mon[108570]: pgmap v64: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:22.273 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:21 smithi042 ceph-mon[113926]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:22.273 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:21 smithi042 ceph-mon[113926]: pgmap v64: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:23.962 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:23 smithi159 ceph-mon[33330]: pgmap v65: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:23 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:23 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[113926]: pgmap v65: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[108570]: pgmap v65: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:24.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:30:24.985 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[108570]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:24.985 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:24.985 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:24.985 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:24.986 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[108570]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:25.155 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:24 smithi159 ceph-mon[33330]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:24 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:24 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:24 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:24 smithi159 ceph-mon[33330]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:25.308 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[113926]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:25.309 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:25.309 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:25.309 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:25.309 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:24 smithi042 ceph-mon[113926]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:30:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:25 smithi159 ceph-mon[33330]: pgmap v66: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:25 smithi042 ceph-mon[108570]: pgmap v66: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:25 smithi042 ceph-mon[113926]: pgmap v66: 89 pgs: 89 active+clean; 548 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-09-13T14:30:28.665 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.665 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:30:28.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:28.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:28.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:28.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all mgr 2024-09-13T14:30:28.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:30:28.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:30:28.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:30:28.667 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[108570]: pgmap v67: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:28.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:30:28.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:28.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:28.668 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:28.669 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all mgr 2024-09-13T14:30:28.669 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.669 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:30:28.669 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:30:28.670 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:30:28.670 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:28 smithi042 ceph-mon[113926]: pgmap v67: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: Upgrade: Setting container_image for all mgr 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon ok-to-stop", "ids": ["b"]} : dispatch 2024-09-13T14:30:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: Upgrade: It appears safe to stop mon.b 2024-09-13T14:30:28.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:28 smithi159 ceph-mon[33330]: pgmap v67: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:30:29.244 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:30:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:29] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.51.0" 2024-09-13T14:30:30.365 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:30:30.365 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (3m) 21s ago 10m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:30:30.365 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (2m) 20s ago 10m 81.4M - c8b91775d855 3a8b5443f86c 2024-09-13T14:30:30.365 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (9m) 21s ago 9m 21.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:30:30.365 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (9m) 20s ago 9m 18.4M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (5m) 20s ago 14m 453M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (5m) 21s ago 16m 510M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (2m) 21s ago 16m 48.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (15m) 20s ago 15m 58.7M 2048M 18.2.4-843-g2c01458a c129f5a2fd4e d2d7d8dd2787 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (106s) 21s ago 14m 34.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (4m) 21s ago 10m 11.0M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (4m) 20s ago 10m 9.87M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (14m) 21s ago 14m 402M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (14m) 21s ago 14m 351M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (13m) 21s ago 13m 490M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (13m) 21s ago 13m 502M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (12m) 20s ago 12m 429M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:30:30.366 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (12m) 20s ago 12m 459M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:30:30.367 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (12m) 20s ago 12m 319M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:30:30.367 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (11m) 20s ago 11m 398M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:30:30.367 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (4m) 20s ago 11m 45.1M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: Upgrade: Updating mon.b 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: Deploying daemon mon.b on smithi159 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:30 smithi159 ceph-mon[33330]: pgmap v68: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-13T14:30:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: Upgrade: Updating mon.b 2024-09-13T14:30:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: Deploying daemon mon.b on smithi159 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[108570]: pgmap v68: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: Upgrade: Updating mon.b 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='client.24970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: Deploying daemon mon.b on smithi159 2024-09-13T14:30:30.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:30.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:30 smithi042 ceph-mon[113926]: pgmap v68: 89 pgs: 89 active+clean; 297 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-09-13T14:30:31.015 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:30:31.015 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:30:31.015 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 1, 2024-09-13T14:30:31.015 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 11, 2024-09-13T14:30:31.016 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 4 2024-09-13T14:30:31.017 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:30:31.017 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:30:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:31 smithi042 ceph-mon[108570]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1467017883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:31.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:31 smithi042 ceph-mon[113926]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1467017883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 ceph-mon[33330]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 ceph-mon[33330]: from='client.? 172.21.15.42:0/1467017883' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:30:31.582 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/3 daemons upgraded", 2024-09-13T14:30:31.583 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:30:31.583 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:30:31.583 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:30:31.870 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 systemd[1]: Stopping Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:30:31.933 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 131112 objects 2024-09-13T14:30:31.933 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :154.796 2024-09-13T14:30:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b[33298]: 2024-09-13T14:30:31.867+0000 7fa4040dc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:30:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b[33298]: 2024-09-13T14:30:31.867+0000 7fa4040dc640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T14:30:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:31 smithi159 podman[102334]: 2024-09-13 14:30:31.931267957 +0000 UTC m=+0.194617671 container died d2d7d8dd278760503f0cb66928f9956543b123fcb193d1169f23aa3a85e1ed98 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, org.label-schema.vendor=CentOS, RELEASE=reef-2c01458, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T14:30:32.924 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:32 smithi159 podman[102334]: 2024-09-13 14:30:32.722683539 +0000 UTC m=+0.986033314 container cleanup d2d7d8dd278760503f0cb66928f9956543b123fcb193d1169f23aa3a85e1ed98 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T14:30:32.925 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:32 smithi159 bash[102334]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b 2024-09-13T14:30:33.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:32 smithi159 podman[102350]: 2024-09-13 14:30:32.924179294 +0000 UTC m=+0.987577033 container remove d2d7d8dd278760503f0cb66928f9956543b123fcb193d1169f23aa3a85e1ed98 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:30:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T14:30:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 systemd[1]: Stopped Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:30:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service: Consumed 11.598s CPU time. 2024-09-13T14:30:33.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 systemd[1]: Starting Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:30:33.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 podman[102480]: 2024-09-13 14:30:33.453593517 +0000 UTC m=+0.016137257 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:30:33.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 podman[102480]: 2024-09-13 14:30:33.583763135 +0000 UTC m=+0.146306911 container create d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T14:30:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 podman[102480]: 2024-09-13 14:30:33.836262739 +0000 UTC m=+0.398806469 container init d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:30:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 podman[102480]: 2024-09-13 14:30:33.841159495 +0000 UTC m=+0.403703232 container start d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T14:30:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T14:30:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev), process ceph-mon, pid 7 2024-09-13T14:30:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: pidfile_write: ignore empty --pid-file 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: load: jerasure load: lrc 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: RocksDB version: 7.9.2 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Git sha 0 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Compile date 2024-09-12 20:46:14 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: DB SUMMARY 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: DB Session ID: NMYPW321KOZ1AX9IZET7 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: CURRENT file: CURRENT 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1411 Bytes 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-13T14:30:34.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9644337 ; 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.error_if_exists: 0 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.create_if_missing: 0 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.paranoid_checks: 1 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.env: 0x5564d37f4160 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.info_log: 0x5564d5f11880 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.statistics: (nil) 2024-09-13T14:30:34.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.use_fsync: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_log_file_size: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_fallocate: 1 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.use_direct_reads: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.db_log_dir: 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.wal_dir: 2024-09-13T14:30:34.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.write_buffer_manager: 0x5564d5f14960 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T14:30:34.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.unordered_write: 0 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.row_cache: None 2024-09-13T14:30:34.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.wal_filter: None 2024-09-13T14:30:34.255 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.two_write_queues: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.wal_compression: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.atomic_flush: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.log_readahead_size: 0 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T14:30:34.256 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_background_jobs: 2 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_background_compactions: -1 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_subcompactions: 1 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T14:30:34.257 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_open_files: -1 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_background_flushes: -1 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Compression algorithms supported: 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kZSTD supported: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kXpressCompression supported: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kBZip2Compression supported: 0 2024-09-13T14:30:34.258 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kLZ4Compression supported: 1 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kZlibCompression supported: 1 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: kSnappyCompression supported: 1 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.merge_operator: 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_filter: None 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T14:30:34.259 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5564d5f114e0) 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: cache_index_and_filter_blocks: 1 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: pin_top_level_index_and_filter: 1 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: index_type: 0 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: data_block_index_type: 0 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: index_shortening: 1 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: checksum: 4 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: no_block_cache: 0 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_cache: 0x5564d5f04850 2024-09-13T14:30:34.260 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_cache_name: BinnedLRUCache 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_cache_options: 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: capacity : 536870912 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: num_shard_bits : 4 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: strict_capacity_limit : 0 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: high_pri_pool_ratio: 0.000 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_cache_compressed: (nil) 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: persistent_cache: (nil) 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_size: 4096 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_size_deviation: 10 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_restart_interval: 16 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: index_block_restart_interval: 1 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: metadata_block_size: 4096 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: partition_filters: 0 2024-09-13T14:30:34.261 INFO:journalctl@ceph.mon.b.smithi159.stdout: use_delta_encoding: 1 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: filter_policy: bloomfilter 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: whole_key_filtering: 1 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: verify_compression: 0 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: read_amp_bytes_per_bit: 0 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: format_version: 5 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: enable_index_compression: 1 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: block_align: 0 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: max_auto_readahead_size: 262144 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: prepopulate_block_cache: 0 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: initial_auto_readahead_size: 8192 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T14:30:34.262 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression: NoCompression 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.num_levels: 7 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:30:34.263 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T14:30:34.264 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T14:30:34.265 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T14:30:34.266 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.inplace_update_support: 0 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T14:30:34.267 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.bloom_locality: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.max_successive_merges: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.ttl: 2592000 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enable_blob_files: false 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.min_blob_size: 0 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T14:30:34.268 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 12862, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d025369f-fc25-4a25-beda-7721c62ec39c 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237833868905, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237833894233, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5932401, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12867, "largest_seqno": 17515, "table_properties": {"data_size": 5914728, "index_size": 11698, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4933, "raw_key_size": 44116, "raw_average_key_size": 22, "raw_value_size": 5874920, "raw_average_value_size": 2985, "num_data_blocks": 522, "num_entries": 1968, "num_filter_entries": 1968, "num_deletions": 261, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726237833, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d025369f-fc25-4a25-beda-7721c62ec39c", "db_session_id": "NMYPW321KOZ1AX9IZET7", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-13T14:30:34.269 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726237833894364, "job": 1, "event": "recovery_finished"} 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5564d5f34e00 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: DB pointer 0x5564d6034000 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: ** DB Stats ** 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T14:30:34.270 INFO:journalctl@ceph.mon.b.smithi159.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: ** Compaction Stats [default] ** 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: L0 1/0 5.66 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: L6 1/0 7.79 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: Sum 2/0 13.44 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 590.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: ** Compaction Stats [default] ** 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T14:30:34.271 INFO:journalctl@ceph.mon.b.smithi159.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 590.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Cumulative compaction: 0.01 GB write, 197.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Interval compaction: 0.01 GB write, 197.85 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T14:30:34.272 INFO:journalctl@ceph.mon.b.smithi159.stdout: Block cache BinnedLRUCache@0x5564d5f04850#7 capacity: 512.00 MB usage: 17.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: starting mon.b rank 1 at public addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] at bind addrs [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???) e3 preinit fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).mds e9 new map 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).mds e9 print_map 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: e9 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: legacy client fscid: 1 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.273 INFO:journalctl@ceph.mon.b.smithi159.stdout: Filesystem 'foo' (1) 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: fs_name foo 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: epoch 8 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: created 2024-09-13T14:20:45.772517+0000 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: modified 2024-09-13T14:20:58.813521+0000 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: tableserver 0 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: root 0 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: session_timeout 60 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: session_autoclose 300 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: max_file_size 1099511627776 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: max_xattr_size 65536 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: required_client_features {} 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: last_failure 0 2024-09-13T14:30:34.274 INFO:journalctl@ceph.mon.b.smithi159.stdout: last_failure_osd_epoch 0 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: max_mds 1 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: in 0 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: up {0=14652} 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: failed 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: damaged 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: stopped 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: data_pools [4] 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: metadata_pool 3 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: inline_data disabled 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: balancer 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: bal_rank_mask -1 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: standby_count_wanted 1 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: qdb_cluster leader: 0 members: 2024-09-13T14:30:34.275 INFO:journalctl@ceph.mon.b.smithi159.stdout: [mds.foo.smithi042.pkkbnw{0:14652} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.42:6834/4264252763,v1:172.21.15.42:6835/4264252763] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout: Standby daemons: 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout: 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout: [mds.foo.smithi159.ypxvst{-1:24514} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.159:6832/3862994713,v1:172.21.15.159:6833/3862994713] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).osd e85 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).osd e85 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 ceph-mon[102519]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 bash[102480]: d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 2024-09-13T14:30:34.276 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:33 smithi159 systemd[1]: Started Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:30:35.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:35 smithi159 ceph-mon[102519]: from='client.24985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:35 smithi159 ceph-mon[102519]: pgmap v69: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:30:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:35 smithi042 ceph-mon[108570]: from='client.24985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:35 smithi042 ceph-mon[108570]: pgmap v69: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:30:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:35 smithi042 ceph-mon[113926]: from='client.24985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:30:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:35 smithi042 ceph-mon[113926]: pgmap v69: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: mon.a calling monitor election 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: mon.b calling monitor election 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: mon.c calling monitor election 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:30:36.510 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: monmap epoch 4 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: last_changed 2024-09-13T14:30:35.246881+0000 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: min_mon_release 19 (squid) 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: election_strategy: 1 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:30:36.511 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: overall HEALTH_OK 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:36.512 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:36 smithi159 ceph-mon[102519]: pgmap v71: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:30:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:30:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: mon.a calling monitor election 2024-09-13T14:30:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: mon.b calling monitor election 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: mon.c calling monitor election 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: monmap epoch 4 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: last_changed 2024-09-13T14:30:35.246881+0000 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: min_mon_release 19 (squid) 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: election_strategy: 1 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:30:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: overall HEALTH_OK 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[108570]: pgmap v71: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: mon.a calling monitor election 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: mon.b calling monitor election 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: mon.c calling monitor election 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: monmap epoch 4 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: last_changed 2024-09-13T14:30:35.246881+0000 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: created 2024-09-13T14:13:51.959909+0000 2024-09-13T14:30:36.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: min_mon_release 19 (squid) 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: election_strategy: 1 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.a 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: 1: [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] mon.b 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: 2: [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0] mon.c 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: fsmap foo:1 {0=foo.smithi042.pkkbnw=up:active} 1 up:standby 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: mgrmap e42: y(active, since 2m), standbys: x 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: overall HEALTH_OK 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:36.602 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:36 smithi042 ceph-mon[113926]: pgmap v71: 89 pgs: 89 active+clean; 43 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:30:37.220 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-13T14:30:37.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-13T14:30:37.440 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:37 smithi159 ceph-mon[102519]: pgmap v72: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:30:38.330 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:37 smithi042 ceph-mon[113926]: pgmap v72: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:30:38.330 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:37 smithi042 ceph-mon[108570]: pgmap v72: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-09-13T14:30:39.043 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:38 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:30:38.905+0000 7f23e3861640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T14:30:39.916 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_1' removed 2024-09-13T14:30:39.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1713294126' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:39 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1713294126' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:39 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:39] "GET /metrics HTTP/1.1" 200 36147 "" "Prometheus/2.51.0" 2024-09-13T14:30:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1713294126' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:39 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:30:40.994 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:30:40.995 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:30:40.995 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-13T14:30:40.995 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-13T14:30:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:40 smithi159 ceph-mon[102519]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:40 smithi159 ceph-mon[102519]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:30:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:41.307 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:41.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:41.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[108570]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:41.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[108570]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:30:41.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:41.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:41.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:30:41.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[113926]: pgmap v74: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:41.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[113926]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T14:30:41.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:41.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.770 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[108570]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:42.771 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.771 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.771 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:42 smithi042 ceph-mon[113926]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:42.992 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:42 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:42 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:42.993 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:42 smithi159 ceph-mon[102519]: pgmap v75: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:43.523 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_2' created 2024-09-13T14:30:43.535 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1812005650' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:43.535 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:43.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1812005650' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:43.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:43.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1812005650' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:43.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-13T14:30:44.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-13T14:30:44.353 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:30:44.669 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:30:44.669 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:44 smithi159 ceph-mon[102519]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:30:44.669 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:44 smithi159 ceph-mon[102519]: pgmap v77: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:44.819 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:30:44.819 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[108570]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:30:44.819 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[108570]: pgmap v77: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:44.820 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-13T14:30:44.820 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[113926]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T14:30:44.820 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:44 smithi042 ceph-mon[113926]: pgmap v77: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:45.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:45.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:30:45.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[108570]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:45.786 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.787 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:45 smithi042 ceph-mon[113926]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:30:45.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: osdmap e88: 8 total, 8 up, 8 in 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: Reconfiguring mon.a (monmap changed)... 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:45 smithi159 ceph-mon[102519]: Reconfiguring daemon mon.a on smithi042 2024-09-13T14:30:46.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:46 smithi042 ceph-mon[108570]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:30:46.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:46 smithi042 ceph-mon[108570]: pgmap v80: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:46.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:46 smithi042 ceph-mon[113926]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:30:46.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:46 smithi042 ceph-mon[113926]: pgmap v80: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:46.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:46 smithi159 ceph-mon[102519]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T14:30:46.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:46 smithi159 ceph-mon[102519]: pgmap v80: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:47.949 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[113926]: pgmap v81: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:30:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:48.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:47 smithi042 ceph-mon[108570]: pgmap v81: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: Reconfiguring mgr.y (monmap changed)... 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: Reconfiguring daemon mgr.y on smithi042 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:30:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:47 smithi159 ceph-mon[102519]: pgmap v81: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:48 smithi159 ceph-mon[102519]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:30:49.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:49.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:48 smithi042 ceph-mon[113926]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:30:49.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/93979831' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:30:49.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:48 smithi042 ceph-mon[108570]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T14:30:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:49] "GET /metrics HTTP/1.1" 200 36147 "" "Prometheus/2.51.0" 2024-09-13T14:30:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[108570]: pgmap v83: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:30:50.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:49 smithi042 ceph-mon[113926]: pgmap v83: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: Reconfiguring mon.c (monmap changed)... 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: Reconfiguring daemon mon.c on smithi042 2024-09-13T14:30:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:49 smithi159 ceph-mon[102519]: pgmap v83: 89 pgs: 89 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:30:50.563 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-13T14:30:50.566 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:30:50.566 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:30:50.622 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:30:50.622 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:30:50.622 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_120369 2024-09-13T14:30:50.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:30:50.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.0 on smithi042 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:51 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.0 on smithi042 2024-09-13T14:30:51.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:51 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.0 on smithi042 2024-09-13T14:30:51.622 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3523 3507 219.18 219.188 0.00314062 0.00454489 2024-09-13T14:30:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:52 smithi042 ceph-mon[108570]: pgmap v84: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 228 op/s 2024-09-13T14:30:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:52 smithi042 ceph-mon[113926]: pgmap v84: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 228 op/s 2024-09-13T14:30:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:52 smithi159 ceph-mon[102519]: pgmap v84: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s wr, 228 op/s 2024-09-13T14:30:52.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 6920 6904 215.736 212.312 0.00388869 0.0046217 2024-09-13T14:30:53.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 9964 9948 207.234 190.25 0.00915958 0.00481775 2024-09-13T14:30:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:30:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:30:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:54.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 12847 12831 200.468 180.188 0.00318406 0.00497826 2024-09-13T14:30:55.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[113926]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:30:55.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.1 on smithi042 2024-09-13T14:30:55.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[113926]: pgmap v85: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 201 op/s 2024-09-13T14:30:55.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[108570]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:30:55.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.1 on smithi042 2024-09-13T14:30:55.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:54 smithi042 ceph-mon[108570]: pgmap v85: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 201 op/s 2024-09-13T14:30:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:54 smithi159 ceph-mon[102519]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T14:30:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:54 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.1 on smithi042 2024-09-13T14:30:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:54 smithi159 ceph-mon[102519]: pgmap v85: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 201 op/s 2024-09-13T14:30:55.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 15647 15631 195.371 175 0.00628417 0.00511204 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.2 on smithi042 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:56 smithi159 ceph-mon[102519]: pgmap v86: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2024-09-13T14:30:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.2 on smithi042 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[108570]: pgmap v86: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2024-09-13T14:30:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.2 on smithi042 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:56.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:56 smithi042 ceph-mon[113926]: pgmap v86: 89 pgs: 89 active+clean; 106 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s wr, 168 op/s 2024-09-13T14:30:56.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 18368 18352 191.151 170.062 0.00497953 0.00522371 2024-09-13T14:30:57.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 20690 20674 184.574 145.125 0.00695272 0.00541188 2024-09-13T14:30:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[108570]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-13T14:30:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:30:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[113926]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-13T14:30:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:30:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:57 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:57 smithi159 ceph-mon[102519]: pgmap v87: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-13T14:30:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:57 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:57 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:30:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:57 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:30:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:57 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:30:58.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 23039 23023 179.853 146.812 0.00649318 0.00555213 2024-09-13T14:30:59.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:58 smithi159 ceph-mon[102519]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:30:59.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:30:58 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.3 on smithi042 2024-09-13T14:30:59.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:58 smithi042 ceph-mon[113926]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:30:59.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:30:58 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.3 on smithi042 2024-09-13T14:30:59.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:58 smithi042 ceph-mon[108570]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T14:30:59.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:30:58 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.3 on smithi042 2024-09-13T14:30:59.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 25126 25110 174.361 130.438 0.00578195 0.00572924 2024-09-13T14:30:59.986 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:30:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:30:59] "GET /metrics HTTP/1.1" 200 36147 "" "Prometheus/2.51.0" 2024-09-13T14:31:00.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:00 smithi042 ceph-mon[108570]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:31:00.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:00 smithi042 ceph-mon[113926]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:31:00.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:00 smithi159 ceph-mon[102519]: pgmap v88: 89 pgs: 89 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-13T14:31:00.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 26954 26938 168.349 114.25 0.00438907 0.00593332 2024-09-13T14:31:01.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 29253 29237 166.106 143.688 0.0110525 0.00601376 2024-09-13T14:31:02.248 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:31:02.623 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 31051 31035 161.627 112.375 0.00600026 0.00618127 2024-09-13T14:31:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: Reconfiguring mds.foo.smithi042.pkkbnw (monmap changed)... 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: Reconfiguring daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[113926]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: Reconfiguring mds.foo.smithi042.pkkbnw (monmap changed)... 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: Reconfiguring daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:31:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:02 smithi042 ceph-mon[108570]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-13T14:31:03.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: Reconfiguring mds.foo.smithi042.pkkbnw (monmap changed)... 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi042.pkkbnw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: Reconfiguring daemon mds.foo.smithi042.pkkbnw on smithi042 2024-09-13T14:31:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:02 smithi159 ceph-mon[102519]: pgmap v89: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (4m) 54s ago 11m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (3m) 22s ago 11m 81.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (10m) 54s ago 10m 21.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (10m) 22s ago 10m 19.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:31:03.385 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (5m) 22s ago 15m 454M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (6m) 54s ago 16m 510M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (2m) 54s ago 17m 48.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (29s) 22s ago 15m 26.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (2m) 54s ago 15m 34.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (5m) 54s ago 10m 11.0M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (5m) 22s ago 10m 10.2M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (14m) 54s ago 14m 402M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (14m) 54s ago 14m 351M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (14m) 54s ago 14m 490M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (13m) 54s ago 13m 502M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (13m) 22s ago 13m 432M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (13m) 22s ago 13m 461M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:31:03.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (12m) 22s ago 12m 318M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:31:03.387 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (12m) 22s ago 12m 399M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:31:03.387 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (4m) 22s ago 11m 45.1M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:31:03.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 32704 32688 157.141 103.312 0.0117923 0.00635778 2024-09-13T14:31:03.992 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:31:03.992 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:31:03.993 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 10, 2024-09-13T14:31:03.994 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 5 2024-09-13T14:31:03.994 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:31:03.994 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:31:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:03 smithi042 ceph-mon[113926]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:03 smithi042 ceph-mon[113926]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:03 smithi042 ceph-mon[108570]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:03 smithi042 ceph-mon[108570]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:03 smithi159 ceph-mon[102519]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:03 smithi159 ceph-mon[102519]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:04.524 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:31:04.524 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:31:04.524 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "mon" 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "3/3 daemons upgraded", 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:31:04.525 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:31:04.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 34361 34345 153.313 103.562 0.00255751 0.00651741 2024-09-13T14:31:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[113926]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[113926]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3499604334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[108570]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[108570]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3499604334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:05.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:04 smithi159 ceph-mon[102519]: from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:05.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:04 smithi159 ceph-mon[102519]: pgmap v90: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3499604334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:05.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 36109 36093 150.375 109.25 0.00982539 0.00664451 2024-09-13T14:31:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:05 smithi042 ceph-mon[113926]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:05 smithi042 ceph-mon[108570]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:06.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:05 smithi159 ceph-mon[102519]: from='client.34233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:06.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:06.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 38092 38076 148.722 123.938 0.00835748 0.00671803 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: pgmap v91: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:31:06.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: pgmap v91: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:31:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: pgmap v91: 89 pgs: 89 active+clean; 1.7 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T14:31:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:07.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 40075 40059 147.263 123.938 0.0138539 0.00678561 2024-09-13T14:31:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:07 smithi159 ceph-mon[102519]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:31:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:07 smithi159 ceph-mon[102519]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:31:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:07 smithi042 ceph-mon[113926]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:31:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:07 smithi042 ceph-mon[113926]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:31:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:07 smithi042 ceph-mon[108570]: Reconfiguring mon.b (monmap changed)... 2024-09-13T14:31:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:07 smithi042 ceph-mon[108570]: Reconfiguring daemon mon.b on smithi159 2024-09-13T14:31:08.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 42012 41996 145.807 121.062 0.00892994 0.00685298 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 184 MiB/s wr, 2.95k op/s 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:31:08.943 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 184 MiB/s wr, 2.95k op/s 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:31:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 184 MiB/s wr, 2.95k op/s 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T14:31:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:09.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 43910 43894 144.376 118.625 0.00606885 0.00692083 2024-09-13T14:31:09.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:09 smithi159 ceph-mon[102519]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:31:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:09 smithi159 ceph-mon[102519]: Reconfiguring daemon mgr.x on smithi159 2024-09-13T14:31:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:09 smithi042 ceph-mon[108570]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:31:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:09 smithi042 ceph-mon[108570]: Reconfiguring daemon mgr.x on smithi159 2024-09-13T14:31:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:09] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-13T14:31:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:09 smithi042 ceph-mon[113926]: Reconfiguring mgr.x (monmap changed)... 2024-09-13T14:31:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:09 smithi042 ceph-mon[113926]: Reconfiguring daemon mgr.x on smithi159 2024-09-13T14:31:10.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:31:10.624566+0000 min lat: 0.00133912 max lat: 0.156046 avg lat: 0.00699059 2024-09-13T14:31:10.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:10.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 45764 45748 142.95 115.875 0.0071793 0.00699059 2024-09-13T14:31:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:10 smithi159 ceph-mon[102519]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:31:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:31:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[113926]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:31:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:31:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[108570]: pgmap v93: 89 pgs: 89 active+clean; 2.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-13T14:31:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T14:31:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:11.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 47403 47387 141.02 102.438 0.0118126 0.00708556 2024-09-13T14:31:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:11 smithi042 ceph-mon[113926]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:31:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:11 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.4 on smithi159 2024-09-13T14:31:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:11 smithi042 ceph-mon[108570]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:31:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:11 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.4 on smithi159 2024-09-13T14:31:12.122 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:11 smithi159 ceph-mon[102519]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T14:31:12.122 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:11 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.4 on smithi159 2024-09-13T14:31:12.624 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 48852 48836 138.726 90.5625 0.0112725 0.00720211 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[113926]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[108570]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-13T14:31:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:31:13.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:12 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:13.157 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:12 smithi159 ceph-mon[102519]: pgmap v94: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 154 MiB/s wr, 2.47k op/s 2024-09-13T14:31:13.157 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:12 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.157 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:12 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:13.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:12 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T14:31:13.158 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:12 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:13.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 50475 50459 137.105 101.438 0.0128712 0.00728881 2024-09-13T14:31:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:13 smithi042 ceph-mon[113926]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:31:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:13 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.5 on smithi159 2024-09-13T14:31:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:13 smithi042 ceph-mon[108570]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:31:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:13 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.5 on smithi159 2024-09-13T14:31:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:13 smithi159 ceph-mon[102519]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T14:31:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:13 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.5 on smithi159 2024-09-13T14:31:14.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 52172 52156 135.811 106.062 0.00657727 0.00735735 2024-09-13T14:31:15.116 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:14 smithi159 ceph-mon[102519]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:15.116 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.116 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.116 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:31:15.116 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[108570]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:31:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[113926]: pgmap v95: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T14:31:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:15.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 53847 53831 134.566 104.688 0.00759734 0.00742597 2024-09-13T14:31:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:15 smithi159 ceph-mon[102519]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:31:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:15 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.6 on smithi159 2024-09-13T14:31:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:15 smithi042 ceph-mon[113926]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:31:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:15 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.6 on smithi159 2024-09-13T14:31:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:15 smithi042 ceph-mon[108570]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T14:31:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:15 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.6 on smithi159 2024-09-13T14:31:16.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 55483 55467 133.322 102.25 0.00615252 0.007495 2024-09-13T14:31:17.136 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:16 smithi159 ceph-mon[102519]: pgmap v96: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:17.136 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.136 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.136 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:31:17.136 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[113926]: pgmap v96: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:31:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:17.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[108570]: pgmap v96: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-13T14:31:17.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:17.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T14:31:17.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:17.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 57104 57088 132.136 101.312 0.00506265 0.00756195 2024-09-13T14:31:18.108 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:17 smithi159 ceph-mon[102519]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:31:18.108 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:17 smithi159 ceph-mon[102519]: Reconfiguring daemon osd.7 on smithi159 2024-09-13T14:31:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:17 smithi042 ceph-mon[113926]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:31:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:17 smithi042 ceph-mon[113926]: Reconfiguring daemon osd.7 on smithi159 2024-09-13T14:31:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:17 smithi042 ceph-mon[108570]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T14:31:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:17 smithi042 ceph-mon[108570]: Reconfiguring daemon osd.7 on smithi159 2024-09-13T14:31:18.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 58677 58661 130.928 98.3125 0.00968784 0.00763268 2024-09-13T14:31:19.076 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:18 smithi159 ceph-mon[102519]: pgmap v97: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:31:19.076 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.077 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.077 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:19.077 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[113926]: pgmap v97: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:31:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[108570]: pgmap v97: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2024-09-13T14:31:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi159.ypxvst", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T14:31:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:19.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 60231 60215 129.762 97.125 0.00807815 0.00770131 2024-09-13T14:31:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:19 smithi042 ceph-mon[113926]: Reconfiguring mds.foo.smithi159.ypxvst (monmap changed)... 2024-09-13T14:31:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:19 smithi042 ceph-mon[113926]: Reconfiguring daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:31:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:19 smithi042 ceph-mon[108570]: Reconfiguring mds.foo.smithi159.ypxvst (monmap changed)... 2024-09-13T14:31:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:19 smithi042 ceph-mon[108570]: Reconfiguring daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:31:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:19] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-13T14:31:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:19 smithi159 ceph-mon[102519]: Reconfiguring mds.foo.smithi159.ypxvst (monmap changed)... 2024-09-13T14:31:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:19 smithi159 ceph-mon[102519]: Reconfiguring daemon mds.foo.smithi159.ypxvst on smithi159 2024-09-13T14:31:20.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 61623 61607 128.336 87 0.0170479 0.00778648 2024-09-13T14:31:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: pgmap v98: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:31:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:31:21.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:31:21.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:21.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: pgmap v98: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:31:21.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:31:21.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:31:21.351 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:31:21.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:21.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: pgmap v98: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T14:31:21.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.357 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:21.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:21.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 62949 62933 126.87 82.875 0.0188515 0.00787657 2024-09-13T14:31:22.237 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:31:22.237 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all grafana 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all loki 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all promtail 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Finalizing container_image settings 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[113926]: Upgrade: Complete! 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:31:22.238 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:31:22.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all grafana 2024-09-13T14:31:22.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all loki 2024-09-13T14:31:22.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all promtail 2024-09-13T14:31:22.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Finalizing container_image settings 2024-09-13T14:31:22.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:21 smithi042 ceph-mon[108570]: Upgrade: Complete! 2024-09-13T14:31:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:31:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all grafana 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all loki 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all promtail 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Finalizing container_image settings 2024-09-13T14:31:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:21 smithi159 ceph-mon[102519]: Upgrade: Complete! 2024-09-13T14:31:22.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 64285 64269 125.514 83.5 0.0124586 0.00796182 2024-09-13T14:31:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:22 smithi159 ceph-mon[102519]: pgmap v99: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T14:31:23.260 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:22 smithi042 ceph-mon[113926]: pgmap v99: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T14:31:23.260 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:22 smithi042 ceph-mon[108570]: pgmap v99: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-09-13T14:31:23.625 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 65460 65444 123.936 73.4375 0.0405197 0.0080623 2024-09-13T14:31:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:23 smithi159 ceph-mon[102519]: pgmap v100: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:23 smithi042 ceph-mon[108570]: pgmap v100: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:23 smithi042 ceph-mon[113926]: pgmap v100: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:24.626 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 66657 66641 122.491 74.8125 0.0179992 0.00815815 2024-09-13T14:31:25.626 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 67953 67937 121.305 81 0.00387762 0.00823802 2024-09-13T14:31:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:25 smithi159 ceph-mon[102519]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:25 smithi042 ceph-mon[113926]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:25 smithi042 ceph-mon[108570]: pgmap v101: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T14:31:26.627 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 69264 69248 120.212 81.9375 0.00966552 0.00831383 2024-09-13T14:31:27.626 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 70386 70370 118.858 70.125 0.02131 0.008409 2024-09-13T14:31:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:27 smithi159 ceph-mon[102519]: pgmap v102: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:31:28.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:27 smithi042 ceph-mon[108570]: pgmap v102: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:31:28.290 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:27 smithi042 ceph-mon[113926]: pgmap v102: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2024-09-13T14:31:28.626 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 71639 71623 117.79 78.3125 0.0206235 0.00848421 2024-09-13T14:31:29.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 72907 72891 116.797 79.25 0.00912033 0.00855681 2024-09-13T14:31:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:29] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.51.0" 2024-09-13T14:31:30.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:31:30.628134+0000 min lat: 0.00133912 max lat: 0.156046 avg lat: 0.00864077 2024-09-13T14:31:30.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:30.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 74044 74028 115.653 71.0625 0.0205875 0.00864077 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: pgmap v103: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: pgmap v103: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:31:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: pgmap v103: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:31:31.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:31:31.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 75324 75308 114.784 80 0.0105768 0.00870712 2024-09-13T14:31:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:31 smithi159 ceph-mon[102519]: pgmap v104: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:31:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:31 smithi042 ceph-mon[113926]: pgmap v104: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:31:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:31 smithi042 ceph-mon[108570]: pgmap v104: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:31:32.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 76613 76597 113.969 80.5625 0.017724 0.00876898 2024-09-13T14:31:33.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 77907 77891 113.199 80.875 0.0123041 0.00882853 2024-09-13T14:31:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:33 smithi159 ceph-mon[102519]: pgmap v105: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:33 smithi042 ceph-mon[113926]: pgmap v105: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:33 smithi042 ceph-mon[108570]: pgmap v105: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:34.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 79103 79087 112.325 74.75 0.00580794 0.00889715 2024-09-13T14:31:35.628 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 80256 80240 111.43 72.0625 0.0105635 0.0089689 2024-09-13T14:31:36.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:31:36.050 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:36.055 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:36.056 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:36.386 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:31:36.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 81382 81366 110.538 70.375 0.0195655 0.00904131 2024-09-13T14:31:37.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:36 smithi042 ceph-mon[113926]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:36 smithi042 ceph-mon[113926]: pgmap v106: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:37.090 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:36 smithi042 ceph-mon[108570]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.090 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:36 smithi042 ceph-mon[108570]: pgmap v106: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:36 smithi159 ceph-mon[102519]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:31:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:36 smithi159 ceph-mon[102519]: pgmap v106: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-13T14:31:37.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 82558 82542 109.749 73.5 0.00556122 0.00910637 2024-09-13T14:31:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:37 smithi159 ceph-mon[102519]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:31:38.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:37 smithi042 ceph-mon[113926]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:31:38.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:37 smithi042 ceph-mon[108570]: pgmap v107: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-13T14:31:38.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 83730 83714 108.989 73.25 0.00690985 0.00916994 2024-09-13T14:31:39.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 84951 84935 108.322 76.3125 0.0114048 0.00922714 2024-09-13T14:31:39.993 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:39] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.51.0" 2024-09-13T14:31:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:39 smithi159 ceph-mon[102519]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:39 smithi042 ceph-mon[113926]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:39 smithi042 ceph-mon[108570]: pgmap v108: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:40.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 86133 86117 107.633 73.875 0.00800073 0.00928472 2024-09-13T14:31:41.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 87312 87296 106.967 73.6875 0.0142146 0.00934292 2024-09-13T14:31:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:41 smithi159 ceph-mon[102519]: pgmap v109: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:31:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:41 smithi042 ceph-mon[108570]: pgmap v109: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:31:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:41 smithi042 ceph-mon[113926]: pgmap v109: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2024-09-13T14:31:42.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 88395 88379 106.212 67.6875 0.0170239 0.00940957 2024-09-13T14:31:43.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 89398 89382 105.39 62.6875 0.00822249 0.00948274 2024-09-13T14:31:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:43 smithi159 ceph-mon[102519]: pgmap v110: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:44.260 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:43 smithi042 ceph-mon[113926]: pgmap v110: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:44.261 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:43 smithi042 ceph-mon[108570]: pgmap v110: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:44.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 90472 90456 104.682 67.125 0.0139327 0.00954752 2024-09-13T14:31:45.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 91621 91605 104.084 71.8125 0.0177156 0.00960271 2024-09-13T14:31:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:45 smithi159 ceph-mon[102519]: pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:45 smithi042 ceph-mon[113926]: pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:45 smithi042 ceph-mon[108570]: pgmap v111: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T14:31:46.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 92734 92718 103.467 69.5625 0.00402714 0.00965955 2024-09-13T14:31:47.629 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 93850 93834 102.876 69.75 0.0175476 0.00971529 2024-09-13T14:31:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:47 smithi159 ceph-mon[102519]: pgmap v112: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:31:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:47 smithi042 ceph-mon[113926]: pgmap v112: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:31:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:47 smithi042 ceph-mon[108570]: pgmap v112: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-09-13T14:31:48.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 94958 94942 102.296 69.25 0.0109769 0.00977047 2024-09-13T14:31:49.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 96059 96043 101.728 68.8125 0.00942749 0.00982484 2024-09-13T14:31:49.996 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:49] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.51.0" 2024-09-13T14:31:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:49 smithi159 ceph-mon[102519]: pgmap v113: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:31:50.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:49 smithi042 ceph-mon[113926]: pgmap v113: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:31:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:50 smithi042 ceph-mon[108570]: pgmap v113: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-13T14:31:50.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:31:50.629949+0000 min lat: 0.00133912 max lat: 0.156046 avg lat: 0.00988045 2024-09-13T14:31:50.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:31:50.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 97134 97118 101.153 67.1875 0.0140645 0.00988045 2024-09-13T14:31:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:31:51.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 98222 98206 100.609 68 0.00575464 0.00993405 2024-09-13T14:31:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:52 smithi042 ceph-mon[113926]: pgmap v114: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:31:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:52 smithi042 ceph-mon[108570]: pgmap v114: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:31:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:52 smithi159 ceph-mon[102519]: pgmap v114: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-09-13T14:31:52.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 99270 99254 100.043 65.5 0.00617411 0.00999073 2024-09-13T14:31:53.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 100307 100291 99.4834 64.8125 0.00735448 0.0100471 2024-09-13T14:31:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:54 smithi042 ceph-mon[113926]: pgmap v115: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:54 smithi042 ceph-mon[108570]: pgmap v115: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:54.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:54 smithi159 ceph-mon[102519]: pgmap v115: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:54.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 15 101327 101312 98.9259 63.8125 0.021099 0.0101033 2024-09-13T14:31:55.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 102360 102344 98.3962 64.5 0.0114962 0.010158 2024-09-13T14:31:56.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 103402 103386 97.892 65.125 0.0159507 0.0102095 2024-09-13T14:31:57.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:56 smithi159 ceph-mon[102519]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:57.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:56 smithi042 ceph-mon[113926]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:57.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:56 smithi042 ceph-mon[108570]: pgmap v116: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2024-09-13T14:31:57.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 104418 104402 97.3787 63.5 0.00724715 0.0102633 2024-09-13T14:31:58.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 105415 105399 96.8629 62.3125 0.00831026 0.0103182 2024-09-13T14:31:59.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:31:59 smithi042 ceph-mon[113926]: pgmap v117: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:31:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:31:59 smithi042 ceph-mon[108570]: pgmap v117: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:31:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:31:59 smithi159 ceph-mon[102519]: pgmap v117: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:31:59.630 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 106384 106368 96.3367 60.5625 0.0129375 0.010375 2024-09-13T14:32:00.100 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:31:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:31:59] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:32:00.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 107390 107374 95.8586 62.875 0.0132195 0.0104265 2024-09-13T14:32:01.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:01 smithi042 ceph-mon[113926]: pgmap v118: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2024-09-13T14:32:01.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:01 smithi042 ceph-mon[108570]: pgmap v118: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2024-09-13T14:32:01.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:01 smithi159 ceph-mon[102519]: pgmap v118: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 892 op/s 2024-09-13T14:32:01.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 108399 108383 95.3967 63.0625 0.0174785 0.0104766 2024-09-13T14:32:02.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 109410 109394 94.9492 63.1875 0.0176045 0.0105266 2024-09-13T14:32:03.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:03 smithi042 ceph-mon[113926]: pgmap v119: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:32:03.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:03 smithi042 ceph-mon[108570]: pgmap v119: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:32:03.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:03 smithi159 ceph-mon[102519]: pgmap v119: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T14:32:03.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 110397 110381 94.4936 61.6875 0.00660729 0.0105779 2024-09-13T14:32:04.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 111374 111358 94.0417 61.0625 0.0222711 0.0106285 2024-09-13T14:32:05.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:05 smithi042 ceph-mon[113926]: pgmap v120: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:05 smithi042 ceph-mon[108570]: pgmap v120: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:05.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:05 smithi159 ceph-mon[102519]: pgmap v120: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:05.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 112335 112319 93.5886 60.0625 0.0171278 0.0106797 2024-09-13T14:32:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:06.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:06.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 113288 113272 93.1408 59.5625 0.0289089 0.0107307 2024-09-13T14:32:07.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:07 smithi042 ceph-mon[113926]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:07.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:07 smithi042 ceph-mon[108570]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:07.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:07 smithi159 ceph-mon[102519]: pgmap v121: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-09-13T14:32:07.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 114247 114231 92.7096 59.9375 0.0129465 0.0107814 2024-09-13T14:32:08.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 115213 115197 92.295 60.375 0.00662861 0.0108295 2024-09-13T14:32:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:09 smithi042 ceph-mon[113926]: pgmap v122: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:32:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:09 smithi042 ceph-mon[108570]: pgmap v122: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:32:09.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:09 smithi159 ceph-mon[102519]: pgmap v122: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:32:09.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 116169 116153 91.8829 59.75 0.0182143 0.0108779 2024-09-13T14:32:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:09] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.51.0" 2024-09-13T14:32:10.631 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:32:10.631765+0000 min lat: 0.00133912 max lat: 0.156046 avg lat: 0.0109266 2024-09-13T14:32:10.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:32:10.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 117115 117099 91.4734 59.125 0.0359066 0.0109266 2024-09-13T14:32:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:11 smithi042 ceph-mon[113926]: pgmap v123: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:32:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:11 smithi042 ceph-mon[108570]: pgmap v123: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:32:11.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:11 smithi159 ceph-mon[102519]: pgmap v123: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T14:32:11.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 118048 118032 91.0639 58.3125 0.00626095 0.0109757 2024-09-13T14:32:12.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 118990 118974 90.6713 58.875 0.0168545 0.011024 2024-09-13T14:32:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:13 smithi042 ceph-mon[113926]: pgmap v124: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:32:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:13 smithi042 ceph-mon[108570]: pgmap v124: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:32:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:13 smithi159 ceph-mon[102519]: pgmap v124: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:32:13.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 119839 119823 90.2182 53.0625 0.0138788 0.0110784 2024-09-13T14:32:14.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 120634 120618 89.7356 49.6875 0.0155003 0.0111382 2024-09-13T14:32:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:15 smithi042 ceph-mon[113926]: pgmap v125: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:15 smithi042 ceph-mon[108570]: pgmap v125: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:15 smithi159 ceph-mon[102519]: pgmap v125: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:15.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 121474 121458 89.2975 52.5 0.0143884 0.0111934 2024-09-13T14:32:16.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 122368 122352 88.9088 55.875 0.0108255 0.0112408 2024-09-13T14:32:17.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:17 smithi042 ceph-mon[113926]: pgmap v126: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:17.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:17 smithi042 ceph-mon[108570]: pgmap v126: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:17.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:17 smithi159 ceph-mon[102519]: pgmap v126: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s 2024-09-13T14:32:17.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 123277 123261 88.5398 56.8125 0.0102787 0.0112889 2024-09-13T14:32:18.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 124172 124156 88.1693 55.9375 0.0348289 0.0113367 2024-09-13T14:32:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:19 smithi042 ceph-mon[108570]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:32:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:19 smithi042 ceph-mon[113926]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:32:19.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:19 smithi159 ceph-mon[102519]: pgmap v127: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-13T14:32:19.632 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 125074 125058 87.812 56.375 0.0141701 0.0113826 2024-09-13T14:32:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:19] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.51.0" 2024-09-13T14:32:20.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 90 6 125951 125945 87.4522 55.4375 0.0302566 0.0114297 2024-09-13T14:32:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:21 smithi042 ceph-mon[108570]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-09-13T14:32:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:21 smithi042 ceph-mon[113926]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-09-13T14:32:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:21 smithi159 ceph-mon[102519]: pgmap v128: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-09-13T14:32:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0142 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 125951 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 87.4522 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 36.318 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 219.188 2024-09-13T14:32:21.633 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 49.6875 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1399 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 581.088 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3507 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 795 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0114299 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0077898 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 0.156046 2024-09-13T14:32:21.634 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00133912 2024-09-13T14:32:21.641 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:32:23.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:23 smithi042 ceph-mon[108570]: pgmap v129: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:32:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:23 smithi042 ceph-mon[113926]: pgmap v129: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:32:23.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:23 smithi159 ceph-mon[102519]: pgmap v129: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:32:25.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:25 smithi042 ceph-mon[108570]: pgmap v130: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2024-09-13T14:32:25.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:25 smithi042 ceph-mon[113926]: pgmap v130: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2024-09-13T14:32:25.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:25 smithi159 ceph-mon[102519]: pgmap v130: 89 pgs: 89 active+clean; 7.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 43 MiB/s wr, 689 op/s 2024-09-13T14:32:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:27 smithi042 ceph-mon[108570]: pgmap v131: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 753 op/s 2024-09-13T14:32:27.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:27 smithi042 ceph-mon[113926]: pgmap v131: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 753 op/s 2024-09-13T14:32:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:27 smithi159 ceph-mon[102519]: pgmap v131: 89 pgs: 89 active+clean; 7.6 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 753 op/s 2024-09-13T14:32:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:29 smithi042 ceph-mon[113926]: pgmap v132: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 1.04k op/s 2024-09-13T14:32:29.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:29 smithi042 ceph-mon[108570]: pgmap v132: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 1.04k op/s 2024-09-13T14:32:29.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:29 smithi159 ceph-mon[102519]: pgmap v132: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 43 MiB/s wr, 1.04k op/s 2024-09-13T14:32:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:29] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-09-13T14:32:31.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:31 smithi159 ceph-mon[102519]: pgmap v133: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 680 op/s 2024-09-13T14:32:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:31 smithi042 ceph-mon[108570]: pgmap v133: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 680 op/s 2024-09-13T14:32:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:31 smithi042 ceph-mon[113926]: pgmap v133: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 680 op/s 2024-09-13T14:32:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:33.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:33 smithi159 ceph-mon[102519]: pgmap v134: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.05k op/s 2024-09-13T14:32:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:33 smithi042 ceph-mon[108570]: pgmap v134: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.05k op/s 2024-09-13T14:32:33.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:33 smithi042 ceph-mon[113926]: pgmap v134: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 21 MiB/s wr, 1.05k op/s 2024-09-13T14:32:35.424 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:35 smithi042 ceph-mon[113926]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 721 op/s 2024-09-13T14:32:35.424 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:35 smithi042 ceph-mon[108570]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 721 op/s 2024-09-13T14:32:35.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:35 smithi159 ceph-mon[102519]: pgmap v135: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 721 op/s 2024-09-13T14:32:36.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:36.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:36.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:36.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 781 op/s 2024-09-13T14:32:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 781 op/s 2024-09-13T14:32:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: pgmap v136: 89 pgs: 89 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 85 B/s wr, 781 op/s 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:39.733 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:39 smithi042 ceph-mon[113926]: pgmap v137: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:32:39.734 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:39 smithi042 ceph-mon[108570]: pgmap v137: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:32:39.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for all mons to upgrade' 2024-09-13T14:32:39.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:39 smithi159 ceph-mon[102519]: pgmap v137: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-09-13T14:32:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:39] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.51.0" 2024-09-13T14:32:40.283 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:41.973 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:41 smithi042 ceph-mon[113926]: pgmap v138: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:32:41.974 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:41 smithi042 ceph-mon[108570]: pgmap v138: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:32:41.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:41 smithi159 ceph-mon[102519]: pgmap v138: 89 pgs: 89 active+clean; 6.9 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-13T14:32:42.267 INFO:teuthology.orchestra.run.smithi042.stdout:wait for all mons to upgrade 2024-09-13T14:32:43.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-09-13T14:32:43.406 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:43.911 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:43 smithi042 ceph-mon[113926]: pgmap v139: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:32:43.912 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:43 smithi042 ceph-mon[108570]: pgmap v139: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:32:43.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:43 smithi159 ceph-mon[102519]: pgmap v139: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:32:45.713 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:45 smithi042 ceph-mon[113926]: pgmap v140: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:32:45.714 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:45 smithi042 ceph-mon[108570]: pgmap v140: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:32:45.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:45 smithi159 ceph-mon[102519]: pgmap v140: 89 pgs: 89 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-09-13T14:32:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:47 smithi042 ceph-mon[113926]: pgmap v141: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:32:47.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:47 smithi042 ceph-mon[113926]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:47.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:47 smithi042 ceph-mon[108570]: pgmap v141: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:32:47.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:47 smithi042 ceph-mon[108570]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:47.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:47 smithi159 ceph-mon[102519]: pgmap v141: 89 pgs: 89 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:32:47.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:47 smithi159 ceph-mon[102519]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:49.105 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: pgmap v142: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: pgmap v142: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:32:49.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:49.731 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:49.731 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:49.731 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:49.731 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: pgmap v142: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 941 op/s 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:49.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:49] "GET /metrics HTTP/1.1" 200 36222 "" "Prometheus/2.51.0" 2024-09-13T14:32:50.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:32:50.601 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:50.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:50 smithi159 ceph-mon[102519]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:50.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:50 smithi159 ceph-mon[102519]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:50 smithi042 ceph-mon[108570]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:50 smithi042 ceph-mon[108570]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:50 smithi042 ceph-mon[113926]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:50 smithi042 ceph-mon[113926]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:32:51.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:51 smithi042 ceph-mon[108570]: pgmap v143: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-13T14:32:51.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:51.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:51 smithi042 ceph-mon[113926]: pgmap v143: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-13T14:32:51.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:51 smithi159 ceph-mon[102519]: pgmap v143: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s 2024-09-13T14:32:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:32:53.946 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:53 smithi042 ceph-mon[113926]: pgmap v144: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-09-13T14:32:53.946 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:53 smithi042 ceph-mon[108570]: pgmap v144: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-09-13T14:32:53.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:53 smithi159 ceph-mon[102519]: pgmap v144: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-09-13T14:32:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all mon 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all crash 2024-09-13T14:32:55.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all rgw 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all nfs 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:32:55.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all grafana 2024-09-13T14:32:55.003 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all loki 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all promtail 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: Upgrade: Finalizing container_image settings 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:32:55.004 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:32:55.005 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:32:55.006 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:32:55.007 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.007 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:55.007 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:55.007 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:55.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all mon 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all crash 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all rgw 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all nfs 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all grafana 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all loki 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all promtail 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: Upgrade: Finalizing container_image settings 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:32:55.103 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:32:55.104 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:32:55.105 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all mgr 2024-09-13T14:32:55.106 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all mon 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all crash 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all rgw 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-13T14:32:55.107 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all iscsi 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all nfs 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.108 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all node-exporter 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all prometheus 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all alertmanager 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all grafana 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all loki 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all promtail 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: Upgrade: Finalizing container_image settings 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:32:55.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T14:32:55.110 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:32:55.111 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:32:55.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:32:55.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:32:55.112 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:55 smithi159 ceph-mon[102519]: pgmap v145: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2024-09-13T14:32:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:55 smithi159 ceph-mon[102519]: Upgrade: Complete! 2024-09-13T14:32:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:55 smithi159 ceph-mon[102519]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[113926]: pgmap v145: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2024-09-13T14:32:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[113926]: Upgrade: Complete! 2024-09-13T14:32:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[113926]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[108570]: pgmap v145: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 639 op/s 2024-09-13T14:32:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[108570]: Upgrade: Complete! 2024-09-13T14:32:56.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:55 smithi042 ceph-mon[108570]: from='client.44186 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:32:56.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'sleep 60' 2024-09-13T14:32:56.769 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:32:57.093 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:57.093 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:57.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:32:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:57 smithi042 ceph-mon[108570]: pgmap v146: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:32:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:57 smithi042 ceph-mon[113926]: pgmap v146: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:32:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:57 smithi159 ceph-mon[102519]: pgmap v146: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 699 op/s 2024-09-13T14:33:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:32:59 smithi042 ceph-mon[108570]: pgmap v147: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:33:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:32:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:32:59] "GET /metrics HTTP/1.1" 200 36220 "" "Prometheus/2.51.0" 2024-09-13T14:33:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:32:59 smithi042 ceph-mon[113926]: pgmap v147: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:33:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:32:59 smithi159 ceph-mon[102519]: pgmap v147: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:33:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:01 smithi042 ceph-mon[108570]: pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:33:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:01 smithi042 ceph-mon[113926]: pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:33:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:01 smithi159 ceph-mon[102519]: pgmap v148: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:33:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:03 smithi042 ceph-mon[108570]: pgmap v149: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:03 smithi042 ceph-mon[113926]: pgmap v149: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:03 smithi159 ceph-mon[102519]: pgmap v149: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:05 smithi042 ceph-mon[113926]: pgmap v150: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T14:33:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:05 smithi042 ceph-mon[108570]: pgmap v150: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T14:33:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:05 smithi159 ceph-mon[102519]: pgmap v150: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-13T14:33:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:07 smithi042 ceph-mon[113926]: pgmap v151: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:33:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:07 smithi042 ceph-mon[108570]: pgmap v151: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:33:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:07 smithi159 ceph-mon[102519]: pgmap v151: 89 pgs: 89 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 738 op/s 2024-09-13T14:33:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:09 smithi042 ceph-mon[108570]: pgmap v152: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:33:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:09] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:33:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:09 smithi042 ceph-mon[113926]: pgmap v152: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:33:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:09 smithi159 ceph-mon[102519]: pgmap v152: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:33:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:11 smithi159 ceph-mon[102519]: pgmap v153: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:33:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:11 smithi042 ceph-mon[108570]: pgmap v153: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:33:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:11 smithi042 ceph-mon[113926]: pgmap v153: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 691 op/s 2024-09-13T14:33:14.150 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:13 smithi042 ceph-mon[108570]: pgmap v154: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:14.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:13 smithi042 ceph-mon[113926]: pgmap v154: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:13 smithi159 ceph-mon[102519]: pgmap v154: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:16.164 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:15 smithi042 ceph-mon[108570]: pgmap v155: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-13T14:33:16.164 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:15 smithi042 ceph-mon[113926]: pgmap v155: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-13T14:33:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:15 smithi159 ceph-mon[102519]: pgmap v155: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-13T14:33:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:17 smithi159 ceph-mon[102519]: pgmap v156: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:33:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:17 smithi042 ceph-mon[108570]: pgmap v156: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:33:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:17 smithi042 ceph-mon[113926]: pgmap v156: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 764 op/s 2024-09-13T14:33:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:19 smithi042 ceph-mon[108570]: pgmap v157: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:33:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:19 smithi042 ceph-mon[113926]: pgmap v157: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:33:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:19] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:33:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:19 smithi159 ceph-mon[102519]: pgmap v157: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:33:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:21 smithi159 ceph-mon[102519]: pgmap v158: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:33:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:21 smithi042 ceph-mon[113926]: pgmap v158: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:33:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:21 smithi042 ceph-mon[108570]: pgmap v158: 89 pgs: 89 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 681 op/s 2024-09-13T14:33:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:23 smithi159 ceph-mon[102519]: pgmap v159: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:33:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:23 smithi042 ceph-mon[108570]: pgmap v159: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:33:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:23 smithi042 ceph-mon[113926]: pgmap v159: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T14:33:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:25 smithi159 ceph-mon[102519]: pgmap v160: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:33:26.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:25 smithi042 ceph-mon[113926]: pgmap v160: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:33:26.277 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:25 smithi042 ceph-mon[108570]: pgmap v160: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-09-13T14:33:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:27 smithi159 ceph-mon[102519]: pgmap v161: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:33:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:27 smithi042 ceph-mon[108570]: pgmap v161: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:33:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:27 smithi042 ceph-mon[113926]: pgmap v161: 89 pgs: 89 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-09-13T14:33:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:29 smithi042 ceph-mon[113926]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:33:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:29 smithi042 ceph-mon[108570]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:33:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:29] "GET /metrics HTTP/1.1" 200 36219 "" "Prometheus/2.51.0" 2024-09-13T14:33:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:29 smithi159 ceph-mon[102519]: pgmap v162: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 962 op/s 2024-09-13T14:33:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:31 smithi159 ceph-mon[102519]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-09-13T14:33:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:31 smithi042 ceph-mon[108570]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-09-13T14:33:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:31 smithi042 ceph-mon[113926]: pgmap v163: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-09-13T14:33:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:33 smithi159 ceph-mon[102519]: pgmap v164: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:33:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:33 smithi042 ceph-mon[108570]: pgmap v164: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:33:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:33 smithi042 ceph-mon[113926]: pgmap v164: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:33:36.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:35 smithi159 ceph-mon[102519]: pgmap v165: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:35 smithi042 ceph-mon[108570]: pgmap v165: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:36.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:35 smithi042 ceph-mon[113926]: pgmap v165: 89 pgs: 89 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:36.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:37 smithi159 ceph-mon[102519]: pgmap v166: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:33:38.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:37 smithi042 ceph-mon[113926]: pgmap v166: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:33:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:37 smithi042 ceph-mon[108570]: pgmap v166: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-13T14:33:40.108 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:39 smithi042 ceph-mon[108570]: pgmap v167: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:40.109 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:39] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-09-13T14:33:40.109 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:39 smithi042 ceph-mon[113926]: pgmap v167: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:39 smithi159 ceph-mon[102519]: pgmap v167: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-09-13T14:33:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:41 smithi159 ceph-mon[102519]: pgmap v168: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:41 smithi042 ceph-mon[108570]: pgmap v168: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:41 smithi042 ceph-mon[113926]: pgmap v168: 89 pgs: 89 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-09-13T14:33:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:43 smithi159 ceph-mon[102519]: pgmap v169: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:43 smithi042 ceph-mon[108570]: pgmap v169: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:43 smithi042 ceph-mon[113926]: pgmap v169: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-13T14:33:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:45 smithi159 ceph-mon[102519]: pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T14:33:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:45 smithi042 ceph-mon[108570]: pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T14:33:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:45 smithi042 ceph-mon[113926]: pgmap v170: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-09-13T14:33:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:47 smithi159 ceph-mon[102519]: pgmap v171: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:33:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:47 smithi042 ceph-mon[113926]: pgmap v171: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:33:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:47 smithi042 ceph-mon[108570]: pgmap v171: 89 pgs: 89 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2024-09-13T14:33:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:49 smithi042 ceph-mon[108570]: pgmap v172: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:33:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:49 smithi042 ceph-mon[113926]: pgmap v172: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:33:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:49] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-09-13T14:33:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:49 smithi159 ceph-mon[102519]: pgmap v172: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 955 op/s 2024-09-13T14:33:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:33:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:51 smithi159 ceph-mon[102519]: pgmap v173: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T14:33:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:51 smithi042 ceph-mon[108570]: pgmap v173: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T14:33:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:51 smithi042 ceph-mon[113926]: pgmap v173: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 671 op/s 2024-09-13T14:33:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:53 smithi159 ceph-mon[102519]: pgmap v174: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:33:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:53 smithi042 ceph-mon[108570]: pgmap v174: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:33:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:53 smithi042 ceph-mon[113926]: pgmap v174: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:33:55.220 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:55.347 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:55.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:33:56.117 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:55 smithi159 ceph-mon[102519]: pgmap v175: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:33:56.230 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:55 smithi042 ceph-mon[108570]: pgmap v175: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:33:56.230 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:55 smithi042 ceph-mon[113926]: pgmap v175: 89 pgs: 89 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-09-13T14:33:58.240 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:57 smithi042 ceph-mon[108570]: pgmap v176: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:33:58.240 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:57 smithi042 ceph-mon[113926]: pgmap v176: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:33:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:57 smithi159 ceph-mon[102519]: pgmap v176: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-09-13T14:33:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:33:59 smithi042 ceph-mon[108570]: pgmap v177: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T14:33:59.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:33:59 smithi042 ceph-mon[113926]: pgmap v177: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T14:33:59.456 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:33:59 smithi159 ceph-mon[102519]: pgmap v177: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T14:34:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:33:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:33:59] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-13T14:34:01.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[113926]: pgmap v178: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[108570]: pgmap v178: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:01 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.689 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:01 smithi159 ceph-mon[102519]: pgmap v178: 89 pgs: 89 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-09-13T14:34:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:01 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:01 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:01 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:01 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:02.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:03.869 INFO:teuthology.orchestra.run.smithi042.stdout:wait for half of osds to upgrade 2024-09-13T14:34:04.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:03 smithi042 ceph-mon[108570]: pgmap v179: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:34:04.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:03 smithi042 ceph-mon[113926]: pgmap v179: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:34:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:03 smithi159 ceph-mon[102519]: pgmap v179: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:34:04.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-09-13T14:34:04.956 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:06.071 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:05 smithi042 ceph-mon[108570]: pgmap v180: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:06.071 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:06.072 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:05 smithi042 ceph-mon[113926]: pgmap v180: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:06.072 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:05 smithi159 ceph-mon[102519]: pgmap v180: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:07 smithi159 ceph-mon[102519]: pgmap v181: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:34:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:07 smithi042 ceph-mon[108570]: pgmap v181: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:34:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:07 smithi042 ceph-mon[113926]: pgmap v181: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-09-13T14:34:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:08 smithi159 ceph-mon[102519]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:09.177 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:08 smithi042 ceph-mon[113926]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:09.177 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:08 smithi042 ceph-mon[108570]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:09.986 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:09 smithi042 ceph-mon[108570]: pgmap v182: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:34:09.987 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:09] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.51.0" 2024-09-13T14:34:09.987 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:09 smithi042 ceph-mon[113926]: pgmap v182: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:34:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:09 smithi159 ceph-mon[102519]: pgmap v182: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-09-13T14:34:10.540 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: pgmap v183: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[108570]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: pgmap v183: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:11.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:11.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:11 smithi042 ceph-mon[113926]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -e sha1=d58f73069a8157a9ace12a0247ee3e8497707ea9 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-13T14:34:11.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: pgmap v183: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 643 op/s 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:11 smithi159 ceph-mon[102519]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:d58f73069a8157a9ace12a0247ee3e8497707ea9 2024-09-13T14:34:12.308 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:34:13.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:13 smithi042 ceph-mon[108570]: pgmap v184: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:34:13.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:13 smithi042 ceph-mon[113926]: pgmap v184: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:34:13.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:13 smithi159 ceph-mon[102519]: pgmap v184: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: pgmap v185: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T14:34:16.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:34:16.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:34:16.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all mgr 2024-09-13T14:34:16.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all mon 2024-09-13T14:34:16.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all crash 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[113926]: Upgrade: osd.0 is safe to restart 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all mgr 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all mon 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all crash 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:34:16.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:16 smithi042 ceph-mon[108570]: Upgrade: osd.0 is safe to restart 2024-09-13T14:34:16.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: Target is version 19.3.0-4697-gd58f7306 (squid) 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, digests ['quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f'] 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all mgr 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all mon 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all crash 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T14:34:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:16 smithi159 ceph-mon[102519]: Upgrade: osd.0 is safe to restart 2024-09-13T14:34:17.310 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:34:17.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:17 smithi042 ceph-mon[113926]: pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:34:17.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:17 smithi042 ceph-mon[108570]: pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:34:17.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:17 smithi159 ceph-mon[102519]: pgmap v186: 89 pgs: 89 active+clean; 2.2 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-09-13T14:34:18.473 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:34:18.473 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (7m) 17s ago 14m 17.3M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:34:18.473 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (6m) 18s ago 14m 81.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:34:18.473 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (13m) 17s ago 13m 23.0M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (13m) 18s ago 13m 19.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (8m) 18s ago 18m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (9m) 17s ago 20m 518M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (6m) 17s ago 20m 60.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (3m) 18s ago 18m 41.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (5m) 17s ago 18m 43.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:34:18.474 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (8m) 17s ago 14m 11.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (8m) 18s ago 14m 11.2M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (18m) 17s ago 18m 614M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 71ca55d80d4a 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (17m) 17s ago 17m 476M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (17m) 17s ago 17m 556M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (17m) 17s ago 17m 647M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (16m) 18s ago 16m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (16m) 18s ago 16m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (15m) 18s ago 15m 442M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (15m) 18s ago 15m 468M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:34:18.475 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (7m) 18s ago 14m 47.5M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:34:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[113926]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[113926]: from='client.44201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:18.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:18.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:18.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[108570]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[108570]: from='client.44201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:18.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:18.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:18 smithi159 ceph-mon[102519]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:18 smithi159 ceph-mon[102519]: from='client.44201 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T14:34:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:19.155 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:34:19.155 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:34:19.155 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:34:19.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:19.155 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 10, 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 5 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:34:19.156 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:34:19.741 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:34:19.741 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:34:19.741 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "0/8 daemons upgraded", 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:34:19.742 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:34:19.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[113926]: pgmap v187: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[113926]: Upgrade: Updating osd.0 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[113926]: Deploying daemon osd.0 on smithi042 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[113926]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3640943308' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[108570]: pgmap v187: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[108570]: Upgrade: Updating osd.0 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[108570]: Deploying daemon osd.0 on smithi042 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[108570]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:19.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3640943308' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:19.836 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:19] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.51.0" 2024-09-13T14:34:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:19 smithi159 ceph-mon[102519]: pgmap v187: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2024-09-13T14:34:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:19 smithi159 ceph-mon[102519]: Upgrade: Updating osd.0 2024-09-13T14:34:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:19 smithi159 ceph-mon[102519]: Deploying daemon osd.0 on smithi042 2024-09-13T14:34:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:19 smithi159 ceph-mon[102519]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3640943308' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:20.755 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:20 smithi042 ceph-mon[108570]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:20.755 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:20 smithi042 ceph-mon[113926]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:20 smithi159 ceph-mon[102519]: from='client.34266 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:21.680 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:21 smithi042 ceph-mon[108570]: pgmap v188: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-09-13T14:34:21.680 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:21.680 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:21 smithi042 ceph-mon[113926]: pgmap v188: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-09-13T14:34:21.680 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:21.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:21 smithi159 ceph-mon[102519]: pgmap v188: 89 pgs: 89 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2024-09-13T14:34:21.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:22.288 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:22 smithi042 systemd[1]: Stopping Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:34:23.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[44979]: 2024-09-13T14:34:22.815+0000 7f97a3e4e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:34:23.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[44979]: 2024-09-13T14:34:22.815+0000 7f97a3e4e640 -1 osd.0 90 *** Got signal Terminated *** 2024-09-13T14:34:23.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[44979]: 2024-09-13T14:34:22.815+0000 7f97a3e4e640 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:34:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:23 smithi042 ceph-mon[108570]: pgmap v189: 89 pgs: 89 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-13T14:34:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:23 smithi042 ceph-mon[108570]: osd.0 marked itself down and dead 2024-09-13T14:34:23.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:23 smithi042 ceph-mon[113926]: pgmap v189: 89 pgs: 89 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-13T14:34:23.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:23 smithi042 ceph-mon[113926]: osd.0 marked itself down and dead 2024-09-13T14:34:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:23 smithi159 ceph-mon[102519]: pgmap v189: 89 pgs: 89 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 923 op/s 2024-09-13T14:34:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:23 smithi159 ceph-mon[102519]: osd.0 marked itself down and dead 2024-09-13T14:34:24.392 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:24 smithi042 podman[125827]: 2024-09-13 14:34:24.115505889 +0000 UTC m=+1.476367937 container died 71ca55d80d4aa089165b7de9b1dd874ae23bd46a785bea26e34edd419950d634 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T14:34:24.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:24 smithi042 ceph-mon[108570]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:24.793 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:24 smithi042 ceph-mon[108570]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:34:24.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:24 smithi042 ceph-mon[113926]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:24.793 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:24 smithi042 ceph-mon[113926]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:34:24.793 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:24 smithi042 podman[125827]: 2024-09-13 14:34:24.490084911 +0000 UTC m=+1.850946957 container cleanup 71ca55d80d4aa089165b7de9b1dd874ae23bd46a785bea26e34edd419950d634 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T14:34:24.793 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:24 smithi042 bash[125827]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:34:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:24 smithi159 ceph-mon[102519]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:34:24.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:24 smithi159 ceph-mon[102519]: osdmap e91: 8 total, 7 up, 8 in 2024-09-13T14:34:25.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:24 smithi042 podman[125846]: 2024-09-13 14:34:24.792041097 +0000 UTC m=+0.670798149 container remove 71ca55d80d4aa089165b7de9b1dd874ae23bd46a785bea26e34edd419950d634 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T14:34:25.697 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:25 smithi042 ceph-mon[113926]: pgmap v191: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-13T14:34:25.697 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:25 smithi042 ceph-mon[113926]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:34:25.698 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:25 smithi042 podman[125927]: 2024-09-13 14:34:25.271008412 +0000 UTC m=+0.038246704 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:25.698 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:25 smithi042 podman[125927]: 2024-09-13 14:34:25.385230745 +0000 UTC m=+0.152469032 container create 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:34:25.698 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:25 smithi042 ceph-mon[108570]: pgmap v191: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-13T14:34:25.698 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:25 smithi042 ceph-mon[108570]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:34:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:25 smithi159 ceph-mon[102519]: pgmap v191: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-09-13T14:34:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:25 smithi159 ceph-mon[102519]: osdmap e92: 8 total, 7 up, 8 in 2024-09-13T14:34:26.004 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:25 smithi042 podman[125927]: 2024-09-13 14:34:25.69706834 +0000 UTC m=+0.464306629 container init 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:34:26.004 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:25 smithi042 podman[125927]: 2024-09-13 14:34:25.708663672 +0000 UTC m=+0.475901958 container start 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T14:34:26.004 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:25 smithi042 podman[125927]: 2024-09-13 14:34:25.813682906 +0000 UTC m=+0.580921191 container attach 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:34:26.348 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:26 smithi042 podman[125974]: 2024-09-13 14:34:26.049345738 +0000 UTC m=+0.036247106 container died 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2) 2024-09-13T14:34:27.089 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:26 smithi042 podman[125974]: 2024-09-13 14:34:26.727564545 +0000 UTC m=+0.714465900 container remove 4462b048ba3e136f4d31d79d63135f261d9bc3705b4098400794563dfd0b5465 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T14:34:27.089 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:26 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:34:27.089 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:26 smithi042 systemd[1]: Stopped Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:34:27.089 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:26 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service: Consumed 20min 22.068s CPU time. 2024-09-13T14:34:27.348 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:27 smithi042 systemd[1]: Starting Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:34:27.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:27 smithi042 ceph-mon[113926]: pgmap v193: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-09-13T14:34:27.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:27 smithi042 ceph-mon[108570]: pgmap v193: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-09-13T14:34:27.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:27 smithi042 podman[126071]: 2024-09-13 14:34:27.324467584 +0000 UTC m=+0.034957698 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:27.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:27 smithi042 podman[126071]: 2024-09-13 14:34:27.503959569 +0000 UTC m=+0.214449674 container create 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:34:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:27 smithi159 ceph-mon[102519]: pgmap v193: 89 pgs: 15 stale+active+clean, 74 active+clean; 1.8 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-09-13T14:34:28.052 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:27 smithi042 podman[126071]: 2024-09-13 14:34:27.949131138 +0000 UTC m=+0.659621243 container init 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T14:34:28.052 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:27 smithi042 podman[126071]: 2024-09-13 14:34:27.956895673 +0000 UTC m=+0.667385774 container start 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T14:34:28.348 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 podman[126071]: 2024-09-13 14:34:28.048985317 +0000 UTC m=+0.759475440 container attach 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T14:34:28.348 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:34:28.661 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:34:28.661 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:34:28.661 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:34:28.666 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T14:34:28.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:28 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 12984/77517 objects degraded (16.750%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:29.074 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 12984/77517 objects degraded (16.750%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:29.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 12984/77517 objects degraded (16.750%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T14:34:29.075 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[126082]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:34:29.076 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 bash[126071]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T14:34:29.076 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:28 smithi042 podman[126071]: 2024-09-13 14:34:28.717156192 +0000 UTC m=+1.427646300 container died 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-13T14:34:29.728 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:29 smithi042 ceph-mon[113926]: pgmap v194: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:29.729 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:29 smithi042 podman[126222]: 2024-09-13 14:34:29.421266816 +0000 UTC m=+0.691860963 container remove 8be7557c5d92ecf1ee14c77c8c059ad1658d8a255f063b9e4bc64303332e5da9 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T14:34:29.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:29 smithi042 ceph-mon[108570]: pgmap v194: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:29.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:29 smithi159 ceph-mon[102519]: pgmap v194: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:29] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.51.0" 2024-09-13T14:34:30.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:29 smithi042 podman[126279]: 2024-09-13 14:34:29.662132417 +0000 UTC m=+0.037440621 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:34:30.099 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:29 smithi042 podman[126279]: 2024-09-13 14:34:29.797519957 +0000 UTC m=+0.172828148 container create 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:34:30.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:30 smithi042 podman[126279]: 2024-09-13 14:34:30.100659739 +0000 UTC m=+0.475967942 container init 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:34:30.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:30 smithi042 podman[126279]: 2024-09-13 14:34:30.11249799 +0000 UTC m=+0.487806183 container start 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T14:34:30.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:30 smithi042 bash[126279]: 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 2024-09-13T14:34:30.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:30 smithi042 systemd[1]: Started Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:34:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[113926]: pgmap v195: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:31.599 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:34:31.260+0000 7f8bef640640 -1 Falling back to public interface 2024-09-13T14:34:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[108570]: pgmap v195: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:31 smithi159 ceph-mon[102519]: pgmap v195: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 440 op/s; 12984/77517 objects degraded (16.750%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:33.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:33 smithi042 ceph-mon[113926]: pgmap v196: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:33 smithi042 ceph-mon[108570]: pgmap v196: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:33.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:33 smithi159 ceph-mon[102519]: pgmap v196: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:35 smithi042 ceph-mon[108570]: pgmap v197: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:35.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 10348/61710 objects degraded (16.769%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:35.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:35 smithi042 ceph-mon[113926]: pgmap v197: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:35.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 10348/61710 objects degraded (16.769%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:35.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:35 smithi159 ceph-mon[102519]: pgmap v197: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:35.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 10348/61710 objects degraded (16.769%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[108570]: pgmap v198: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[113926]: pgmap v198: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:37 smithi159 ceph-mon[102519]: pgmap v198: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 1.3 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 10348/61710 objects degraded (16.769%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:40.039 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:39 smithi042 ceph-mon[113926]: pgmap v199: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7763/46344 objects degraded (16.751%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:40.039 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:39 smithi042 ceph-mon[108570]: pgmap v199: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7763/46344 objects degraded (16.751%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:40.040 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:39] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.51.0" 2024-09-13T14:34:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:39 smithi159 ceph-mon[102519]: pgmap v199: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.16k op/s; 7763/46344 objects degraded (16.751%); 0 B/s, 0 objects/s recovering 2024-09-13T14:34:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7763/46344 objects degraded (16.751%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7763/46344 objects degraded (16.751%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7763/46344 objects degraded (16.751%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:41 smithi042 ceph-mon[113926]: pgmap v200: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7763/46344 objects degraded (16.751%) 2024-09-13T14:34:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:41 smithi042 ceph-mon[108570]: pgmap v200: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7763/46344 objects degraded (16.751%) 2024-09-13T14:34:42.253 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:41 smithi159 ceph-mon[102519]: pgmap v200: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 964 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7763/46344 objects degraded (16.751%) 2024-09-13T14:34:42.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:34:42.314+0000 7f8bef640640 -1 osd.0 90 log_to_monitors true 2024-09-13T14:34:43.080 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:42 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:34:43.081 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:42 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:34:43.081 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:34:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:34:42.837+0000 7f8be73ea640 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:34:43.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:42 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T14:34:43.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:34:43.079+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:34:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: pgmap v201: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 755 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5831/36297 objects degraded (16.065%) 2024-09-13T14:34:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: pgmap v201: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 755 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5831/36297 objects degraded (16.065%) 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:44.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:43 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: pgmap v201: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 755 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 5831/36297 objects degraded (16.065%) 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: osdmap e93: 8 total, 7 up, 8 in 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:43 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232] boot 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:34:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232] boot 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:34:45.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:44 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:34:45.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:34:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T14:34:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:34:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: osd.0 [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232] boot 2024-09-13T14:34:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T14:34:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:44 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T14:34:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[113926]: pgmap v204: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[113926]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:34:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5536/32781 objects degraded (16.888%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[108570]: pgmap v204: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[108570]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:34:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5536/32781 objects degraded (16.888%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:45 smithi159 ceph-mon[102519]: pgmap v204: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:45 smithi159 ceph-mon[102519]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T14:34:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5536/32781 objects degraded (16.888%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:46 smithi042 ceph-mon[113926]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:34:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:46 smithi042 ceph-mon[108570]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:34:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:46 smithi159 ceph-mon[102519]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T14:34:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:47 smithi042 ceph-mon[113926]: pgmap v207: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:47 smithi042 ceph-mon[108570]: pgmap v207: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:47 smithi159 ceph-mon[102519]: pgmap v207: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 682 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 5536/32781 objects degraded (16.888%) 2024-09-13T14:34:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:49 smithi042 ceph-mon[113926]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 10 objects/s recovering 2024-09-13T14:34:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:49 smithi042 ceph-mon[108570]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 10 objects/s recovering 2024-09-13T14:34:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:49] "GET /metrics HTTP/1.1" 200 36345 "" "Prometheus/2.51.0" 2024-09-13T14:34:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:49 smithi159 ceph-mon[102519]: pgmap v208: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 10 objects/s recovering 2024-09-13T14:34:50.363 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:34:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8321/24036 objects degraded (34.619%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8321/24036 objects degraded (34.619%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8321/24036 objects degraded (34.619%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:34:51.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:34:51.513 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:34:51.513 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (7m) 15s ago 15m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:34:51.513 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (7m) 51s ago 14m 81.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:34:51.513 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (13m) 15s ago 13m 23.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:34:51.513 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (13m) 51s ago 13m 19.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (9m) 51s ago 19m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (10m) 15s ago 20m 519M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (6m) 15s ago 20m 61.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (4m) 51s ago 19m 41.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (6m) 15s ago 19m 44.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (9m) 15s ago 14m 11.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (8m) 51s ago 14m 11.2M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (21s) 15s ago 18m 14.5M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (18m) 15s ago 18m 479M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (17m) 15s ago 17m 562M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (17m) 15s ago 17m 651M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (17m) 51s ago 17m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (16m) 51s ago 16m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:34:51.514 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (16m) 51s ago 16m 442M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:34:51.515 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (16m) 51s ago 16m 468M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:34:51.515 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (8m) 51s ago 15m 47.5M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:34:52.207 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:34:52.207 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:34:52.207 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:34:52.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:52.207 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 7, 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 9, 2024-09-13T14:34:52.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 6 2024-09-13T14:34:52.209 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:34:52.209 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:34:52.209 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[113926]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 8 objects/s recovering 2024-09-13T14:34:52.209 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[113926]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.209 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[113926]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.209 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[108570]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 8 objects/s recovering 2024-09-13T14:34:52.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[108570]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.210 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:51 smithi042 ceph-mon[108570]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:51 smithi159 ceph-mon[102519]: pgmap v209: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 500 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 468 op/s; 8321/24036 objects degraded (34.619%); 0 B/s, 8 objects/s recovering 2024-09-13T14:34:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:51 smithi159 ceph-mon[102519]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:51 smithi159 ceph-mon[102519]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:34:52.772 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/8 daemons upgraded", 2024-09-13T14:34:52.773 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:34:52.773 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:34:52.773 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:34:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:52 smithi042 ceph-mon[113926]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3560980697' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:52 smithi042 ceph-mon[108570]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3560980697' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:52 smithi159 ceph-mon[102519]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3560980697' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:34:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:53 smithi159 ceph-mon[102519]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 342 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8321/16458 objects degraded (50.559%); 0 B/s, 6 objects/s recovering 2024-09-13T14:34:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:53 smithi159 ceph-mon[102519]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:53 smithi042 ceph-mon[108570]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 342 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8321/16458 objects degraded (50.559%); 0 B/s, 6 objects/s recovering 2024-09-13T14:34:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:53 smithi042 ceph-mon[108570]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:53 smithi042 ceph-mon[113926]: pgmap v210: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 342 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8321/16458 objects degraded (50.559%); 0 B/s, 6 objects/s recovering 2024-09-13T14:34:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:53 smithi042 ceph-mon[113926]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:34:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:55 smithi159 ceph-mon[102519]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 22 objects/s recovering 2024-09-13T14:34:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8321/12888 objects degraded (64.564%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:55 smithi042 ceph-mon[108570]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 22 objects/s recovering 2024-09-13T14:34:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8321/12888 objects degraded (64.564%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:55 smithi042 ceph-mon[113926]: pgmap v211: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 22 objects/s recovering 2024-09-13T14:34:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8321/12888 objects degraded (64.564%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:34:57.122 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 125951 objects 2024-09-13T14:34:57.123 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :155.481 2024-09-13T14:34:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:57 smithi159 ceph-mon[102519]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 20 objects/s recovering 2024-09-13T14:34:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:57 smithi042 ceph-mon[113926]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 20 objects/s recovering 2024-09-13T14:34:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:57 smithi042 ceph-mon[108570]: pgmap v212: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 267 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 8321/12888 objects degraded (64.564%); 0 B/s, 20 objects/s recovering 2024-09-13T14:34:58.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:34:58.095+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:34:59.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:34:59.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:34:59.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[108570]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 33 objects/s recovering 2024-09-13T14:35:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:35:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:34:59] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.51.0" 2024-09-13T14:35:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[113926]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 33 objects/s recovering 2024-09-13T14:35:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:34:59 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:35:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:59 smithi159 ceph-mon[102519]: pgmap v213: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 870 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 33 objects/s recovering 2024-09-13T14:35:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:59 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:34:59 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:35:00.722 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-13T14:35:00.723 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-13T14:35:00.974 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:01.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:00 smithi042 ceph-mon[108570]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:35:01.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8321/1422 objects degraded (585.162%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:01.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:00 smithi042 ceph-mon[113926]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:35:01.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8321/1422 objects degraded (585.162%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:00 smithi159 ceph-mon[102519]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T14:35:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8321/1422 objects degraded (585.162%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:02.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:01 smithi042 ceph-mon[108570]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 35 objects/s recovering 2024-09-13T14:35:02.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:01 smithi042 ceph-mon[108570]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:35:02.188 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:01 smithi042 ceph-mon[113926]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 35 objects/s recovering 2024-09-13T14:35:02.188 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:01 smithi042 ceph-mon[113926]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:35:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:01 smithi159 ceph-mon[102519]: pgmap v215: 89 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 29 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 8321/1422 objects degraded (585.162%); 0 B/s, 35 objects/s recovering 2024-09-13T14:35:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:01 smithi159 ceph-mon[102519]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T14:35:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:02 smithi159 ceph-mon[102519]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:35:03.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:02 smithi042 ceph-mon[108570]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:35:03.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:02 smithi042 ceph-mon[113926]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T14:35:03.906 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_2' removed 2024-09-13T14:35:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:03 smithi159 ceph-mon[102519]: pgmap v218: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5979/153 objects degraded (3907.843%); 0 B/s, 204 objects/s recovering 2024-09-13T14:35:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:03 smithi159 ceph-mon[102519]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:35:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:35:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[108570]: pgmap v218: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5979/153 objects degraded (3907.843%); 0 B/s, 204 objects/s recovering 2024-09-13T14:35:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[108570]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:35:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:35:04.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[113926]: pgmap v218: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.1 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 5979/153 objects degraded (3907.843%); 0 B/s, 204 objects/s recovering 2024-09-13T14:35:04.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[113926]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T14:35:04.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:35:04.783 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:35:04.784 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:35:04.784 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-13T14:35:04.785 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-13T14:35:04.988 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:05.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:35:05.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:04 smithi042 ceph-mon[108570]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:35:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:35:05.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:04 smithi042 ceph-mon[113926]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:35:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1382722944' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:35:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:04 smithi159 ceph-mon[102519]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T14:35:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:05 smithi159 ceph-mon[102519]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:05 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5979/153 objects degraded (3907.843%), 5 pgs degraded) 2024-09-13T14:35:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:05 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:35:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[108570]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5979/153 objects degraded (3907.843%), 5 pgs degraded) 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[113926]: pgmap v221: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5979/153 objects degraded (3907.843%), 5 pgs degraded) 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:07.917 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_3' created 2024-09-13T14:35:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:07 smithi159 ceph-mon[102519]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:35:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:07 smithi042 ceph-mon[108570]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:35:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:07 smithi042 ceph-mon[113926]: pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-13T14:35:09.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:35:09.004 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:08 smithi042 ceph-mon[113926]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:35:09.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:35:09.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:08 smithi042 ceph-mon[108570]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:35:09.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-13T14:35:09.240 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:35:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1167489249' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-13T14:35:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:08 smithi159 ceph-mon[102519]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T14:35:10.052 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[113926]: pgmap v224: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:10.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[113926]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:35:10.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[108570]: pgmap v224: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:10.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-mon[108570]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:35:10.054 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:09] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.51.0" 2024-09-13T14:35:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:09 smithi159 ceph-mon[102519]: pgmap v224: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:09 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:35:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:09 smithi159 ceph-mon[102519]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T14:35:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:10 smithi159 ceph-mon[102519]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:35:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:10 smithi042 ceph-mon[108570]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:35:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:10 smithi042 ceph-mon[113926]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T14:35:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:11.947 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-13T14:35:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:11 smithi159 ceph-mon[102519]: pgmap v227: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:11 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3389792931' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:11 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[113926]: pgmap v227: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3389792931' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[108570]: pgmap v227: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3389792931' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:12.287 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:11 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:35:13.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:12 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:12 smithi159 ceph-mon[102519]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:35:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:12 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:12 smithi042 ceph-mon[108570]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:35:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:12 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:35:13.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:12 smithi042 ceph-mon[113926]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T14:35:13.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:35:13.112+0000 7f23e3861640 -1 mgr.server reply reply (11) Resource temporarily unavailable 6 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:35:13.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:35:13.112+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy 6 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-09-13T14:35:13.683 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-13T14:35:13.686 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:35:13.686 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:35:13.744 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:35:13.744 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:35:13.744 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_130700 2024-09-13T14:35:13.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:13.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:35:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:13 smithi159 ceph-mon[102519]: pgmap v229: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:13 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:13 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:35:14.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:13 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:14.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[113926]: pgmap v229: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:35:14.251 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:14.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[108570]: pgmap v229: 89 pgs: 6 unknown, 83 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:14.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:35:14.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:35:14.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:13 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:14.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3759 3743 233.924 233.938 0.00515343 0.00426243 2024-09-13T14:35:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:14 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:14 smithi042 ceph-mon[113926]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:35:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:14 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:14 smithi042 ceph-mon[108570]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:35:15.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:14 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:14 smithi159 ceph-mon[102519]: Upgrade: 6 pgs have unknown state; cannot draw any conclusions 2024-09-13T14:35:15.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 7095 7079 221.203 208.5 0.00570844 0.00451155 2024-09-13T14:35:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:15 smithi159 ceph-mon[102519]: pgmap v230: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:15 smithi042 ceph-mon[108570]: pgmap v230: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:15 smithi042 ceph-mon[113926]: pgmap v230: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:16.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 10316 10300 214.566 201.312 0.00320265 0.00465359 2024-09-13T14:35:17.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 13649 13633 212.998 208.312 0.00417509 0.0046889 2024-09-13T14:35:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:17 smithi159 ceph-mon[102519]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:17 smithi042 ceph-mon[113926]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:17 smithi042 ceph-mon[108570]: pgmap v231: 89 pgs: 89 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:35:18.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 16770 16754 209.407 195.062 0.00444356 0.00476784 2024-09-13T14:35:19.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 19781 19765 205.868 188.188 0.00905617 0.00485206 2024-09-13T14:35:19.982 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:19] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.51.0" 2024-09-13T14:35:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:19 smithi159 ceph-mon[102519]: pgmap v232: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:35:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:19 smithi042 ceph-mon[113926]: pgmap v232: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:35:20.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:19 smithi042 ceph-mon[108570]: pgmap v232: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:35:20.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 22578 22562 201.429 174.812 0.00831267 0.0049587 2024-09-13T14:35:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:21.745 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 25267 25251 197.257 168.062 0.00883997 0.00506385 2024-09-13T14:35:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:21 smithi159 ceph-mon[102519]: pgmap v233: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:35:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:21 smithi042 ceph-mon[108570]: pgmap v233: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:35:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:21 smithi042 ceph-mon[113926]: pgmap v233: 89 pgs: 89 active+clean; 641 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:35:22.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 15 27787 27772 192.845 157.562 0.00422031 0.005179 2024-09-13T14:35:23.378 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:35:23.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 29752 29736 185.834 122.75 0.0131783 0.00537459 2024-09-13T14:35:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:23 smithi159 ceph-mon[102519]: pgmap v234: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T14:35:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:23 smithi042 ceph-mon[113926]: pgmap v234: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T14:35:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:23 smithi042 ceph-mon[108570]: pgmap v234: 89 pgs: 89 active+clean; 1.2 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 122 MiB/s wr, 1.96k op/s 2024-09-13T14:35:24.558 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:35:24.558 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (8m) 48s ago 15m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:35:24.558 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (7m) 84s ago 15m 81.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (14m) 48s ago 14m 23.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (14m) 84s ago 14m 19.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (10m) 84s ago 19m 455M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (10m) 48s ago 21m 519M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (7m) 48s ago 21m 61.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (4m) 84s ago 19m 41.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (6m) 48s ago 19m 44.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (9m) 48s ago 15m 11.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (9m) 84s ago 15m 11.2M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (54s) 48s ago 19m 14.5M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (18m) 48s ago 18m 479M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e 9b6a46257931 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (18m) 48s ago 18m 562M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:35:24.559 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (18m) 48s ago 18m 651M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:35:24.560 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (17m) 84s ago 17m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:35:24.560 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (17m) 84s ago 17m 498M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:35:24.560 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (16m) 84s ago 16m 442M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:35:24.560 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (16m) 84s ago 16m 468M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:35:24.560 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (8m) 84s ago 16m 47.5M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:35:24.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 31906 31890 181.178 134.625 0.00481171 0.00551288 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:35:25.256 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 7, 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 1 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 9, 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 6 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:35:25.257 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:35:25.260 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:25 smithi042 ceph-mon[113926]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.260 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:25 smithi042 ceph-mon[113926]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.260 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:25 smithi042 ceph-mon[108570]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.260 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:25 smithi042 ceph-mon[108570]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:25 smithi159 ceph-mon[102519]: from='client.54208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:25 smithi159 ceph-mon[102519]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:25.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 15 33973 33958 176.849 129.25 0.0113482 0.00564926 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/8 daemons upgraded", 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:35:25.838 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:35:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[113926]: pgmap v235: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[113926]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3630789850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[113926]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[108570]: pgmap v235: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[108570]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3630789850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:26 smithi042 ceph-mon[108570]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:26 smithi159 ceph-mon[102519]: pgmap v235: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:26 smithi159 ceph-mon[102519]: from='client.34335 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:26 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3630789850' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:26 smithi159 ceph-mon[102519]: from='client.54223 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:26.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 36169 36153 173.798 137.188 0.00251159 0.00574886 2024-09-13T14:35:27.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:27 smithi042 ceph-mon[113926]: pgmap v236: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:27 smithi042 ceph-mon[108570]: pgmap v236: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:27 smithi159 ceph-mon[102519]: pgmap v236: 89 pgs: 89 active+clean; 1.5 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-13T14:35:27.746 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 38392 38376 171.307 138.938 0.00651739 0.00583178 2024-09-13T14:35:28.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 40321 40305 167.919 120.562 0.00681983 0.00595011 2024-09-13T14:35:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[113926]: pgmap v237: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.00k op/s 2024-09-13T14:35:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[113926]: Upgrade: osd.1 is safe to restart 2024-09-13T14:35:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[108570]: pgmap v237: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.00k op/s 2024-09-13T14:35:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-mon[108570]: Upgrade: osd.1 is safe to restart 2024-09-13T14:35:29.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:29 smithi159 ceph-mon[102519]: pgmap v237: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 188 MiB/s wr, 3.00k op/s 2024-09-13T14:35:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T14:35:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:29 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T14:35:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:29 smithi159 ceph-mon[102519]: Upgrade: osd.1 is safe to restart 2024-09-13T14:35:29.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 42317 42301 165.22 124.75 0.00851512 0.00604683 2024-09-13T14:35:30.042 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:29] "GET /metrics HTTP/1.1" 200 36332 "" "Prometheus/2.51.0" 2024-09-13T14:35:30.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 44222 44206 162.504 119.062 0.00888056 0.00614934 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: pgmap v238: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: Upgrade: Updating osd.1 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:31 smithi159 ceph-mon[102519]: Deploying daemon osd.1 on smithi042 2024-09-13T14:35:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: pgmap v238: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-09-13T14:35:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: Upgrade: Updating osd.1 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[108570]: Deploying daemon osd.1 on smithi042 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: pgmap v238: 89 pgs: 89 active+clean; 2.2 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 134 MiB/s wr, 2.15k op/s 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: Upgrade: Updating osd.1 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:31.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:31 smithi042 ceph-mon[113926]: Deploying daemon osd.1 on smithi042 2024-09-13T14:35:31.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 46084 46068 159.941 116.375 0.00966782 0.00624716 2024-09-13T14:35:32.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 47905 47889 157.513 113.812 0.00324458 0.00634249 2024-09-13T14:35:33.392 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:33 smithi042 ceph-mon[113926]: pgmap v239: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:35:33.392 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:33 smithi042 ceph-mon[108570]: pgmap v239: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:35:33.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:33 smithi159 ceph-mon[102519]: pgmap v239: 89 pgs: 89 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-13T14:35:33.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:35:33.747316+0000 min lat: 0.00113323 max lat: 0.130578 avg lat: 0.00643015 2024-09-13T14:35:33.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:33.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 49745 49729 155.387 115 0.00661226 0.00643015 2024-09-13T14:35:34.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 51534 51518 153.311 111.812 0.0111009 0.00651805 2024-09-13T14:35:35.394 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:35 smithi042 ceph-mon[113926]: pgmap v240: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:35:35.395 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:35 smithi042 ceph-mon[108570]: pgmap v240: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:35:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:35 smithi159 ceph-mon[102519]: pgmap v240: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-13T14:35:35.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 53151 53135 150.936 101.062 0.00654049 0.00661986 2024-09-13T14:35:35.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:35 smithi042 systemd[1]: Stopping Ceph osd.1 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:35:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:36 smithi042 ceph-mon[108570]: osd.1 marked itself down and dead 2024-09-13T14:35:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:36 smithi042 ceph-mon[113926]: osd.1 marked itself down and dead 2024-09-13T14:35:36.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[50965]: 2024-09-13T14:35:35.875+0000 7fa660b78640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:35:36.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[50965]: 2024-09-13T14:35:35.875+0000 7fa660b78640 -1 osd.1 105 *** Got signal Terminated *** 2024-09-13T14:35:36.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[50965]: 2024-09-13T14:35:35.875+0000 7fa660b78640 -1 osd.1 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:35:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:36 smithi159 ceph-mon[102519]: osd.1 marked itself down and dead 2024-09-13T14:35:36.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 53837 53821 146.237 42.875 0.00170966 0.00665174 2024-09-13T14:35:37.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:36 smithi042 podman[131292]: 2024-09-13 14:35:36.725899641 +0000 UTC m=+0.999723140 container died 9b6a462579315611df02592d898b2f5c9e6ca6e959c715c243f46cc4de80d28f (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-13T14:35:37.402 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[113926]: pgmap v241: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:35:37.402 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[113926]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:37.402 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[113926]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T14:35:37.403 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:37 smithi042 podman[131292]: 2024-09-13 14:35:37.158029725 +0000 UTC m=+1.431853225 container cleanup 9b6a462579315611df02592d898b2f5c9e6ca6e959c715c243f46cc4de80d28f (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, RELEASE=reef-2c01458, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T14:35:37.403 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:37 smithi042 bash[131292]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1 2024-09-13T14:35:37.403 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[108570]: pgmap v241: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:35:37.403 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[108570]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:37.403 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:37 smithi042 ceph-mon[108570]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T14:35:37.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:37 smithi159 ceph-mon[102519]: pgmap v241: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2024-09-13T14:35:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:37 smithi159 ceph-mon[102519]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:35:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:37 smithi159 ceph-mon[102519]: osdmap e106: 8 total, 7 up, 8 in 2024-09-13T14:35:37.747 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 55007 54991 143.191 73.125 0.00566337 0.00697841 2024-09-13T14:35:37.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:37 smithi042 podman[131310]: 2024-09-13 14:35:37.401827538 +0000 UTC m=+0.673078076 container remove 9b6a462579315611df02592d898b2f5c9e6ca6e959c715c243f46cc4de80d28f (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:35:38.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:38 smithi042 ceph-mon[113926]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T14:35:38.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:38 smithi042 ceph-mon[108570]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T14:35:38.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:37.976299721 +0000 UTC m=+0.026920867 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:38.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:38.103107093 +0000 UTC m=+0.153728212 container create e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:35:38.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:38 smithi159 ceph-mon[102519]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T14:35:38.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 56946 56930 142.31 121.188 0.0125651 0.00702199 2024-09-13T14:35:38.972 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:38.665475221 +0000 UTC m=+0.716096360 container init e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:35:38.973 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:38.675551881 +0000 UTC m=+0.726173011 container start e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:35:38.973 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:38.808984949 +0000 UTC m=+0.859606111 container attach e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T14:35:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[113926]: pgmap v244: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 3372/160941 objects degraded (2.095%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[108570]: pgmap v244: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:39.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 3372/160941 objects degraded (2.095%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 conmon[131411]: conmon e1f20fd4d9472f7eab4b : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183.scope/memory.events 2024-09-13T14:35:39.349 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:38 smithi042 podman[131396]: 2024-09-13 14:35:38.974373263 +0000 UTC m=+1.024994391 container died e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T14:35:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:39 smithi159 ceph-mon[102519]: pgmap v244: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 137 MiB/s wr, 2.20k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:39 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 5 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2024-09-13T14:35:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:39 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 3372/160941 objects degraded (2.095%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:39.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 58919 58903 141.579 123.312 0.0135283 0.00705821 2024-09-13T14:35:40.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:39 smithi042 podman[131439]: 2024-09-13 14:35:39.929419561 +0000 UTC m=+0.944352496 container remove e1f20fd4d9472f7eab4b9f3ce56863104e837824ba0e044a0b44209954e24183 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T14:35:40.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:39 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T14:35:40.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:39 smithi042 systemd[1]: Stopped Ceph osd.1 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:35:40.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:39 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.1.service: Consumed 17min 20.526s CPU time. 2024-09-13T14:35:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:39] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.51.0" 2024-09-13T14:35:40.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:40 smithi042 systemd[1]: Starting Ceph osd.1 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:35:40.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 60797 60781 140.682 117.375 0.015565 0.00710308 2024-09-13T14:35:41.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:40 smithi042 podman[131544]: 2024-09-13 14:35:40.515320654 +0000 UTC m=+0.040357239 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:41.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:40 smithi042 podman[131544]: 2024-09-13 14:35:40.648133562 +0000 UTC m=+0.173170128 container create 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:35:41.432 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 podman[131544]: 2024-09-13 14:35:41.201650107 +0000 UTC m=+0.726686700 container init 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T14:35:41.433 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 podman[131544]: 2024-09-13 14:35:41.209448502 +0000 UTC m=+0.734485072 container start 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T14:35:41.433 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 podman[131544]: 2024-09-13 14:35:41.301436968 +0000 UTC m=+0.826473541 container attach 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T14:35:41.433 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-mon[108570]: pgmap v245: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:41.433 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-mon[113926]: pgmap v245: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:41 smithi159 ceph-mon[102519]: pgmap v245: 89 pgs: 3 active+undersized, 18 peering, 5 stale+active+clean, 2 active+undersized+degraded, 61 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 3372/160941 objects degraded (2.095%) 2024-09-13T14:35:41.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 62668 62652 139.834 116.938 0.00470776 0.00714675 2024-09-13T14:35:41.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:35:41.849 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:35:41.850 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:35:41.850 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-13T14:35:42.282 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate[131557]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 bash[131544]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 conmon[131557]: conmon 6a1de70e5029db154665 : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5.scope/memory.events 2024-09-13T14:35:42.283 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:41 smithi042 podman[131544]: 2024-09-13 14:35:41.948932764 +0000 UTC m=+1.473969354 container died 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T14:35:42.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 64407 64391 138.76 108.688 0.00934274 0.00720173 2024-09-13T14:35:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:43 smithi159 ceph-mon[102519]: pgmap v246: 89 pgs: 21 active+undersized, 11 peering, 7 active+undersized+degraded, 50 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 11101/174429 objects degraded (6.364%) 2024-09-13T14:35:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:43 smithi042 ceph-mon[108570]: pgmap v246: 89 pgs: 21 active+undersized, 11 peering, 7 active+undersized+degraded, 50 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 11101/174429 objects degraded (6.364%) 2024-09-13T14:35:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:43 smithi042 ceph-mon[113926]: pgmap v246: 89 pgs: 21 active+undersized, 11 peering, 7 active+undersized+degraded, 50 active+clean; 3.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 11101/174429 objects degraded (6.364%) 2024-09-13T14:35:43.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 66128 66112 137.719 107.562 0.00933518 0.00725655 2024-09-13T14:35:44.501 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:44 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 27571/188244 objects degraded (14.646%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:44.501 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:44 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T14:35:44.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:44 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 27571/188244 objects degraded (14.646%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:44.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:44 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T14:35:44.502 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:44 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 27571/188244 objects degraded (14.646%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:44.502 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:44 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T14:35:44.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 67701 67685 136.448 98.3125 0.00324334 0.00732093 2024-09-13T14:35:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:45 smithi159 ceph-mon[102519]: pgmap v247: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 27571/188244 objects degraded (14.646%); 8.0 KiB/s, 0 objects/s recovering 2024-09-13T14:35:45.511 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:45 smithi042 ceph-mon[113926]: pgmap v247: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 27571/188244 objects degraded (14.646%); 8.0 KiB/s, 0 objects/s recovering 2024-09-13T14:35:45.511 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:45 smithi042 ceph-mon[108570]: pgmap v247: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 27571/188244 objects degraded (14.646%); 8.0 KiB/s, 0 objects/s recovering 2024-09-13T14:35:45.512 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:45 smithi042 podman[131682]: 2024-09-13 14:35:45.426066918 +0000 UTC m=+3.467237835 container remove 6a1de70e5029db154665b1288fd9cc8dec6b2f63e68e0bc2bb00525cac9699c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS) 2024-09-13T14:35:45.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 69361 69345 135.426 103.75 0.0063869 0.00737888 2024-09-13T14:35:45.777 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:45 smithi042 podman[131740]: 2024-09-13 14:35:45.649036365 +0000 UTC m=+0.030105221 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:35:46.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:45 smithi042 podman[131740]: 2024-09-13 14:35:45.77723032 +0000 UTC m=+0.158299176 container create 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:35:46.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 podman[131740]: 2024-09-13 14:35:46.197973948 +0000 UTC m=+0.579042817 container init 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T14:35:46.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 podman[131740]: 2024-09-13 14:35:46.206425126 +0000 UTC m=+0.587493971 container start 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T14:35:46.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 ceph-osd[131778]: -- 172.21.15.42:0/1525268540 <== mon.2 v2:172.21.15.42:3301/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x55e185ca7ba0 con 0x55e185e0a000 2024-09-13T14:35:46.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 bash[131740]: 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 2024-09-13T14:35:46.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 systemd[1]: Started Ceph osd.1 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:35:46.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 70907 70891 134.25 96.625 0.0118893 0.00744417 2024-09-13T14:35:47.348 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:46 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T14:35:46.867+0000 7f8378e01640 -1 Falling back to public interface 2024-09-13T14:35:47.748 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 72329 72313 132.915 88.875 0.00713381 0.00751847 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: pgmap v248: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 27571/188244 objects degraded (14.646%); 6.5 KiB/s, 0 objects/s recovering 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: pgmap v248: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 27571/188244 objects degraded (14.646%); 6.5 KiB/s, 0 objects/s recovering 2024-09-13T14:35:47.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: pgmap v248: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 27571/188244 objects degraded (14.646%); 6.5 KiB/s, 0 objects/s recovering 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:47.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:47.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:48.749 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 73883 73867 131.891 97.125 0.0112294 0.00757733 2024-09-13T14:35:49.728 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:49 smithi042 ceph-mon[113926]: pgmap v249: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 31088/213045 objects degraded (14.592%); 5.9 KiB/s, 0 objects/s recovering 2024-09-13T14:35:49.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:49 smithi042 ceph-mon[108570]: pgmap v249: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 31088/213045 objects degraded (14.592%); 5.9 KiB/s, 0 objects/s recovering 2024-09-13T14:35:49.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:49 smithi159 ceph-mon[102519]: pgmap v249: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 110 MiB/s wr, 1.77k op/s; 31088/213045 objects degraded (14.592%); 5.9 KiB/s, 0 objects/s recovering 2024-09-13T14:35:49.749 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 75249 75233 130.599 85.375 0.00660552 0.00765241 2024-09-13T14:35:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:49] "GET /metrics HTTP/1.1" 200 36339 "" "Prometheus/2.51.0" 2024-09-13T14:35:50.682 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 31088/213045 objects degraded (14.592%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:50.682 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 31088/213045 objects degraded (14.592%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:50.722 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 31088/213045 objects degraded (14.592%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:50.751 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 76635 76619 129.402 86.625 0.0237316 0.00772177 2024-09-13T14:35:51.751 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 78038 78022 128.305 87.6875 0.013773 0.00778876 2024-09-13T14:35:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:51 smithi159 ceph-mon[102519]: pgmap v250: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 31088/213045 objects degraded (14.592%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[113926]: pgmap v250: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 31088/213045 objects degraded (14.592%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[108570]: pgmap v250: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s; 31088/213045 objects degraded (14.592%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:52.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:35:52.751 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 79448 79432 127.274 88.125 0.00757708 0.00785238 2024-09-13T14:35:53.751 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:35:53.751629+0000 min lat: 0.00113323 max lat: 1.02874 avg lat: 0.00791808 2024-09-13T14:35:53.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:35:53.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 80802 80786 126.208 84.625 0.0142089 0.00791808 2024-09-13T14:35:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:53 smithi159 ceph-mon[102519]: pgmap v251: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 32006/223932 objects degraded (14.293%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[113926]: pgmap v251: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 32006/223932 objects degraded (14.293%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[108570]: pgmap v251: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s; 32006/223932 objects degraded (14.293%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:54.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 82137 82121 125.164 83.4375 0.0161936 0.00798475 2024-09-13T14:35:55.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 83444 83428 124.129 81.6875 0.00556248 0.00805127 2024-09-13T14:35:56.087 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T14:35:55.947+0000 7f8378e01640 -1 osd.1 105 log_to_monitors true 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[113926]: pgmap v252: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 34237/234552 objects degraded (14.597%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 34237/234552 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[108570]: pgmap v252: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 34237/234552 objects degraded (14.597%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 34237/234552 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:55 smithi159 ceph-mon[102519]: pgmap v252: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s; 34237/234552 objects degraded (14.597%); 5.3 KiB/s, 0 objects/s recovering 2024-09-13T14:35:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 34237/234552 objects degraded (14.597%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T14:35:56.414 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:35:56.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 84857 84841 123.296 88.3125 0.00840501 0.00810495 2024-09-13T14:35:57.098 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 14:35:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T14:35:56.884+0000 7f8370bab640 -1 osd.1 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:35:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:56 smithi042 ceph-mon[113926]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:56 smithi042 ceph-mon[113926]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:56 smithi042 ceph-mon[108570]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:56 smithi042 ceph-mon[108570]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:56 smithi159 ceph-mon[102519]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:56 smithi159 ceph-mon[102519]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T14:35:57.479 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (9m) 3s ago 16m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (8m) 4s ago 16m 80.1M - c8b91775d855 3a8b5443f86c 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (15m) 3s ago 15m 23.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (15m) 4s ago 15m 20.3M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (10m) 4s ago 20m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (11m) 3s ago 21m 520M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (7m) 3s ago 22m 63.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (5m) 4s ago 20m 45.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (7m) 3s ago 20m 47.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (10m) 3s ago 15m 10.6M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (10m) 4s ago 15m 11.1M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:35:57.480 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (87s) 3s ago 19m 297M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (11s) 3s ago 19m 13.0M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (19m) 3s ago 19m 696M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (18m) 3s ago 18m 767M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (18m) 4s ago 18m 528M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (17m) 4s ago 17m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (17m) 4s ago 17m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (17m) 4s ago 17m 475M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:35:57.481 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (9m) 4s ago 16m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:35:57.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 15 86207 86192 122.413 84.4375 0.0154511 0.00816433 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:35:58.046 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:35:58.047 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:35:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: pgmap v253: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 34237/234552 objects degraded (14.597%) 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:57 smithi159 ceph-mon[102519]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: pgmap v253: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 34237/234552 objects degraded (14.597%) 2024-09-13T14:35:58.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:35:58.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[113926]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: pgmap v253: 89 pgs: 28 active+undersized, 11 active+undersized+degraded, 50 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 34237/234552 objects degraded (14.597%) 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: from='osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:35:58.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:57 smithi042 ceph-mon[108570]: from='client.44306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.380 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:35:58.380 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:35:58.380 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:35:58.380 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:35:58.381 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:35:58.381 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:35:58.381 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:35:58.381 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:35:58.381 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:35:58.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 86347 86331 119.885 8.6875 0.00428968 0.00817058 2024-09-13T14:35:58.944 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102] boot 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3577583926' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[108570]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102] boot 2024-09-13T14:35:58.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:35:58.946 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:35:58.946 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3577583926' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:58.946 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:58 smithi042 ceph-mon[113926]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:35:59.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: osd.1 [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102] boot 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3577583926' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:58 smithi159 ceph-mon[102519]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T14:35:59.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:35:59.102+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T14:35:59.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 86347 86331 117.279 0 - 0.00817058 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: pgmap v256: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:59.978 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: pgmap v256: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:35:59.979 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[108570]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:35:59.980 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:35:59] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-13T14:36:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: pgmap v256: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:00.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:35:59 smithi159 ceph-mon[102519]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:00.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:35:59 smithi042 ceph-mon[113926]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T14:36:00.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 87136 87120 115.833 24.6562 0.0117296 0.00862746 2024-09-13T14:36:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:00 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:00 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T14:36:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 36766/253608 objects degraded (14.497%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:01.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T14:36:01.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 36766/253608 objects degraded (14.497%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T14:36:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 36766/253608 objects degraded (14.497%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:01.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 88125 88109 114.708 61.8125 0.0070934 0.00871232 2024-09-13T14:36:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:01 smithi159 ceph-mon[102519]: pgmap v259: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:36:02.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:01 smithi042 ceph-mon[113926]: pgmap v259: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:36:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:01 smithi042 ceph-mon[108570]: pgmap v259: 89 pgs: 3 peering, 26 active+undersized, 10 active+undersized+degraded, 50 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 36766/253608 objects degraded (14.497%) 2024-09-13T14:36:02.752 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 89128 89112 113.646 62.6875 0.0150659 0.00879397 2024-09-13T14:36:03.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 90032 90016 112.503 56.5 0.00405666 0.00888339 2024-09-13T14:36:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:03 smithi159 ceph-mon[102519]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 5 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 30231/258843 objects degraded (11.679%); 897 KiB/s, 14 objects/s recovering 2024-09-13T14:36:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:03 smithi042 ceph-mon[113926]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 5 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 30231/258843 objects degraded (11.679%); 897 KiB/s, 14 objects/s recovering 2024-09-13T14:36:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:03 smithi042 ceph-mon[108570]: pgmap v260: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 5 active+undersized+degraded, 74 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 30231/258843 objects degraded (11.679%); 897 KiB/s, 14 objects/s recovering 2024-09-13T14:36:04.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 91015 90999 111.502 61.4375 0.0101756 0.00896332 2024-09-13T14:36:05.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 91981 91965 110.518 60.375 0.00380344 0.00904051 2024-09-13T14:36:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[113926]: pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 14075/264729 objects degraded (5.317%); 784 KiB/s, 12 objects/s recovering 2024-09-13T14:36:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14075/264729 objects degraded (5.317%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[108570]: pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 14075/264729 objects degraded (5.317%); 784 KiB/s, 12 objects/s recovering 2024-09-13T14:36:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14075/264729 objects degraded (5.317%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:06.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:06 smithi159 ceph-mon[102519]: pgmap v261: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s; 14075/264729 objects degraded (5.317%); 784 KiB/s, 12 objects/s recovering 2024-09-13T14:36:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:06 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14075/264729 objects degraded (5.317%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:36:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:06.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 92948 92932 109.573 60.4375 0.0206931 0.00912051 2024-09-13T14:36:07.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 93919 93903 108.668 60.6875 0.0148289 0.00919652 2024-09-13T14:36:08.300 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:08 smithi159 ceph-mon[102519]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 463 op/s; 14075/264729 objects degraded (5.317%); 608 KiB/s, 9 objects/s recovering 2024-09-13T14:36:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:08 smithi042 ceph-mon[113926]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 463 op/s; 14075/264729 objects degraded (5.317%); 608 KiB/s, 9 objects/s recovering 2024-09-13T14:36:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:08 smithi042 ceph-mon[108570]: pgmap v262: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 29 MiB/s wr, 463 op/s; 14075/264729 objects degraded (5.317%); 608 KiB/s, 9 objects/s recovering 2024-09-13T14:36:08.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 94909 94893 107.817 61.875 0.0143783 0.0092694 2024-09-13T14:36:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:09 smithi042 ceph-mon[113926]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 13831/279066 objects degraded (4.956%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:36:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:09 smithi042 ceph-mon[108570]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 13831/279066 objects degraded (4.956%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:36:09.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:09 smithi159 ceph-mon[102519]: pgmap v263: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s; 13831/279066 objects degraded (4.956%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:36:09.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 95843 95827 106.934 58.375 0.0178713 0.00934391 2024-09-13T14:36:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:09] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-13T14:36:10.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 96727 96711 106.027 55.25 0.00886049 0.0094257 2024-09-13T14:36:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:11 smithi159 ceph-mon[102519]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 13831/279066 objects degraded (4.956%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:36:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:11 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 13831/279066 objects degraded (4.956%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:11 smithi042 ceph-mon[113926]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 13831/279066 objects degraded (4.956%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:36:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:11 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 13831/279066 objects degraded (4.956%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:11 smithi042 ceph-mon[108570]: pgmap v264: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 13831/279066 objects degraded (4.956%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:36:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:11 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 13831/279066 objects degraded (4.956%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:11.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 97675 97659 105.221 59.25 0.0172014 0.00949808 2024-09-13T14:36:12.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 98609 98593 104.427 58.375 0.0180636 0.00957132 2024-09-13T14:36:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:13 smithi042 ceph-mon[113926]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 13578/286260 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:36:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:13 smithi042 ceph-mon[108570]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 13578/286260 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:36:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:13 smithi159 ceph-mon[102519]: pgmap v265: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 13578/286260 objects degraded (4.743%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:36:13.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:36:13.753701+0000 min lat: 0.00113323 max lat: 2.19734 avg lat: 0.00964086 2024-09-13T14:36:13.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:36:13.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 99537 99521 103.653 58 0.00619887 0.00964086 2024-09-13T14:36:14.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:14 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:36:14.116+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:14.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 100330 100314 102.766 49.5625 0.0206641 0.00972542 2024-09-13T14:36:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[108570]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:15.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[113926]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:15 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:15 smithi159 ceph-mon[102519]: pgmap v266: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:15 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:15 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:15.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 101157 101141 101.942 51.6875 0.0262009 0.00980411 2024-09-13T14:36:16.386 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 13578/293391 objects degraded (4.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:16.386 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 13578/293391 objects degraded (4.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 13578/293391 objects degraded (4.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:16.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 101999 101983 101.16 52.625 0.0183977 0.00987924 2024-09-13T14:36:17.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:17 smithi159 ceph-mon[102519]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:17 smithi042 ceph-mon[113926]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:17 smithi042 ceph-mon[108570]: pgmap v267: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 50 MiB/s wr, 795 op/s; 13578/293391 objects degraded (4.628%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:36:17.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 102910 102894 100.469 56.9375 0.00937226 0.00994799 2024-09-13T14:36:18.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 103832 103816 99.8093 57.625 0.00909479 0.0100137 2024-09-13T14:36:19.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:19 smithi159 ceph-mon[102519]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 13367/306462 objects degraded (4.362%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:36:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:19 smithi042 ceph-mon[113926]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 13367/306462 objects degraded (4.362%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:36:19.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:19 smithi042 ceph-mon[108570]: pgmap v268: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 13367/306462 objects degraded (4.362%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:36:19.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 104766 104750 99.1815 58.375 0.0109114 0.0100774 2024-09-13T14:36:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:19] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-13T14:36:20.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 105666 105650 98.5407 56.25 0.0132265 0.0101424 2024-09-13T14:36:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:21 smithi159 ceph-mon[102519]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 13367/306462 objects degraded (4.362%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:21 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 13367/306462 objects degraded (4.362%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[113926]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 13367/306462 objects degraded (4.362%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 13367/306462 objects degraded (4.362%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[108570]: pgmap v269: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s; 13367/306462 objects degraded (4.362%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 13367/306462 objects degraded (4.362%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:21.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:21.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 106597 106581 97.9472 58.1875 0.0252064 0.0102044 2024-09-13T14:36:22.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 107516 107500 97.36 57.4375 0.00690013 0.0102654 2024-09-13T14:36:23.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:23 smithi159 ceph-mon[102519]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 13120/313254 objects degraded (4.188%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:36:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:23 smithi042 ceph-mon[113926]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 13120/313254 objects degraded (4.188%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:36:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:23 smithi042 ceph-mon[108570]: pgmap v270: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 13120/313254 objects degraded (4.188%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:36:23.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 108413 108397 96.77 56.0625 0.00867872 0.0103275 2024-09-13T14:36:24.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 109323 109307 96.2081 56.875 0.016145 0.0103883 2024-09-13T14:36:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:25 smithi159 ceph-mon[102519]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:25.524 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:25 smithi042 ceph-mon[113926]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:25.524 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:25 smithi042 ceph-mon[108570]: pgmap v271: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:25.754 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 110223 110207 95.653 56.25 0.00682873 0.0104489 2024-09-13T14:36:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 13120/320109 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:26.509 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 13120/320109 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:26.510 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 13120/320109 objects degraded (4.099%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:26.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 111125 111109 95.115 56.375 0.0109692 0.0105087 2024-09-13T14:36:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:27 smithi159 ceph-mon[102519]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:27.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:27 smithi042 ceph-mon[113926]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:27.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:27 smithi042 ceph-mon[108570]: pgmap v272: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 46 MiB/s wr, 741 op/s; 13120/320109 objects degraded (4.099%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:36:27.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 112002 111986 94.5703 54.8125 0.0313161 0.0105681 2024-09-13T14:36:28.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 15 112833 112818 94.0027 52 0.0329166 0.0106324 2024-09-13T14:36:28.979 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:36:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[113926]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 12846/333786 objects degraded (3.849%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[113926]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[108570]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 12846/333786 objects degraded (3.849%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[108570]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:29.349 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:36:29.132+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:29.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:29 smithi159 ceph-mon[102519]: pgmap v273: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 12846/333786 objects degraded (3.849%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:29 smithi159 ceph-mon[102519]: from='client.54262 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:29.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 113454 113438 93.2756 38.75 0.00977479 0.010715 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (9m) 36s ago 16m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (8m) 37s ago 16m 80.1M - c8b91775d855 3a8b5443f86c 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (15m) 36s ago 15m 23.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (15m) 37s ago 15m 20.3M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (11m) 37s ago 20m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (11m) 36s ago 22m 520M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (8m) 36s ago 22m 63.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (5m) 37s ago 21m 45.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:36:29.977 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (7m) 36s ago 20m 47.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (10m) 36s ago 16m 10.6M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (10m) 37s ago 16m 11.1M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (2m) 36s ago 20m 297M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (44s) 36s ago 20m 13.0M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (19m) 36s ago 19m 696M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (19m) 36s ago 19m 767M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (18m) 37s ago 18m 528M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (18m) 37s ago 18m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (18m) 37s ago 18m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (17m) 37s ago 17m 475M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:36:29.978 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (10m) 37s ago 17m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:36:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:29] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.51.0" 2024-09-13T14:36:30.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:30 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:30 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:30 smithi159 ceph-mon[102519]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:30 smithi159 ceph-mon[102519]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[113926]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[113926]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T14:36:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[108570]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:30 smithi042 ceph-mon[108570]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:36:30.670 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:36:30.671 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:36:30.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 114247 114231 92.7079 49.5625 0.0227391 0.0107805 2024-09-13T14:36:31.242 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:36:31.242 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:36:31.242 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:36:31.243 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:36:31.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:31 smithi159 ceph-mon[102519]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 12846/333786 objects degraded (3.849%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 12846/333786 objects degraded (3.849%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1615279690' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[113926]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 12846/333786 objects degraded (3.849%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 12846/333786 objects degraded (3.849%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1615279690' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[108570]: pgmap v274: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 47 MiB/s wr, 758 op/s; 12846/333786 objects degraded (3.849%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 12846/333786 objects degraded (3.849%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1615279690' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:36:31.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 114987 114971 92.1122 46.25 0.028487 0.0108505 2024-09-13T14:36:32.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:32 smithi159 ceph-mon[102519]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:32 smithi042 ceph-mon[113926]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:32 smithi042 ceph-mon[108570]: from='client.34389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:36:32.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 115875 115859 91.6487 55.5 0.0222309 0.0109055 2024-09-13T14:36:33.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:33 smithi159 ceph-mon[102519]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 12532/339801 objects degraded (3.688%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:36:33.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:33 smithi042 ceph-mon[113926]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 12532/339801 objects degraded (3.688%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:36:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:33 smithi042 ceph-mon[108570]: pgmap v275: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s; 12532/339801 objects degraded (3.688%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:36:33.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:36:33.755522+0000 min lat: 0.00113323 max lat: 2.19734 avg lat: 0.0109614 2024-09-13T14:36:33.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:36:33.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 116748 116732 91.1851 54.5625 0.0178447 0.0109614 2024-09-13T14:36:34.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 117601 117585 90.7175 53.3125 0.0163202 0.0110173 2024-09-13T14:36:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:35 smithi159 ceph-mon[102519]: pgmap v276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:35 smithi042 ceph-mon[108570]: pgmap v276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:35 smithi042 ceph-mon[113926]: pgmap v276: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:35.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 118452 118436 90.2598 53.1875 0.0179045 0.0110734 2024-09-13T14:36:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:36 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 12532/345420 objects degraded (3.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:36.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:36 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 12532/345420 objects degraded (3.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:36.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:36 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 12532/345420 objects degraded (3.628%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:36.755 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 119315 119299 89.8221 53.9375 0.0215216 0.0111275 2024-09-13T14:36:37.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:37 smithi159 ceph-mon[102519]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:37 smithi042 ceph-mon[113926]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:37.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:37 smithi042 ceph-mon[108570]: pgmap v277: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 12532/345420 objects degraded (3.628%); 3.1 MiB/s, 48 objects/s recovering 2024-09-13T14:36:37.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 120160 120144 89.3815 52.8125 0.0102474 0.011183 2024-09-13T14:36:38.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 121010 120994 88.9549 53.125 0.026357 0.0112362 2024-09-13T14:36:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:39 smithi159 ceph-mon[102519]: pgmap v278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12219/358389 objects degraded (3.409%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:36:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:39 smithi042 ceph-mon[108570]: pgmap v278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12219/358389 objects degraded (3.409%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:36:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:39 smithi042 ceph-mon[113926]: pgmap v278: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 12219/358389 objects degraded (3.409%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:36:39.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 121659 121643 88.3922 40.5625 0.00370854 0.0112603 2024-09-13T14:36:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:39] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-13T14:36:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:40 smithi159 ceph-mon[102519]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:36:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 12219/358389 objects degraded (3.409%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:40 smithi042 ceph-mon[113926]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:36:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 12219/358389 objects degraded (3.409%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:40 smithi042 ceph-mon[108570]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T14:36:40.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 12219/358389 objects degraded (3.409%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:40.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 15 122077 122062 87.6772 26.1875 0.0284951 0.0114 2024-09-13T14:36:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:41 smithi159 ceph-mon[102519]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 12219/358389 objects degraded (3.409%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:36:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:41 smithi159 ceph-mon[102519]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:36:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:41 smithi042 ceph-mon[108570]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 12219/358389 objects degraded (3.409%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:36:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:41 smithi042 ceph-mon[108570]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:36:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:41 smithi042 ceph-mon[113926]: pgmap v280: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s; 12219/358389 objects degraded (3.409%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:36:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:41 smithi042 ceph-mon[113926]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T14:36:41.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 122927 122911 87.2838 53.0625 0.00604409 0.0114513 2024-09-13T14:36:42.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 123756 123740 86.8852 51.8125 0.00830722 0.0115035 2024-09-13T14:36:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:43 smithi159 ceph-mon[102519]: pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 12116/363771 objects degraded (3.331%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:36:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:43 smithi042 ceph-mon[108570]: pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 12116/363771 objects degraded (3.331%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:36:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:43 smithi042 ceph-mon[113926]: pgmap v282: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 12116/363771 objects degraded (3.331%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:36:43.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 90 3 124586 124583 86.5052 52.6875 0.0127764 0.0115549 2024-09-13T14:36:44.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:44 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:44.499 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:44 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:44.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:44 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:36:44.500 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:44 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:36:44.148+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:36:44.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0151 2024-09-13T14:36:44.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 124586 2024-09-13T14:36:44.756 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 86.5036 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 46.8787 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 233.938 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1384 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 750.067 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3743 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 0 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0115551 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0300348 2024-09-13T14:36:44.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 2.19734 2024-09-13T14:36:44.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00113323 2024-09-13T14:36:44.763 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:36:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:45 smithi159 ceph-mon[102519]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 11870/369312 objects degraded (3.214%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:36:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:45 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:45 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:36:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 11870/369312 objects degraded (3.214%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[108570]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 11870/369312 objects degraded (3.214%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:36:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:36:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 11870/369312 objects degraded (3.214%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[113926]: pgmap v283: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s; 11870/369312 objects degraded (3.214%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:36:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:36:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:36:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 11870/369312 objects degraded (3.214%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:47.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:47 smithi159 ceph-mon[102519]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 28 MiB/s wr, 454 op/s; 11870/369312 objects degraded (3.214%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:47.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:47 smithi042 ceph-mon[113926]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 28 MiB/s wr, 454 op/s; 11870/369312 objects degraded (3.214%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:47 smithi042 ceph-mon[108570]: pgmap v284: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 28 MiB/s wr, 454 op/s; 11870/369312 objects degraded (3.214%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:36:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:49 smithi042 ceph-mon[113926]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 115 B/s rd, 36 MiB/s wr, 796 op/s; 11543/368166 objects degraded (3.135%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T14:36:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:49 smithi042 ceph-mon[108570]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 115 B/s rd, 36 MiB/s wr, 796 op/s; 11543/368166 objects degraded (3.135%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T14:36:49.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:49 smithi159 ceph-mon[102519]: pgmap v285: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 115 B/s rd, 36 MiB/s wr, 796 op/s; 11543/368166 objects degraded (3.135%); 4.8 MiB/s, 76 objects/s recovering 2024-09-13T14:36:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:49] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.51.0" 2024-09-13T14:36:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 11543/368166 objects degraded (3.135%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 11543/368166 objects degraded (3.135%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:50.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 11543/368166 objects degraded (3.135%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:51.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:51 smithi042 ceph-mon[113926]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 703 op/s; 11543/368166 objects degraded (3.135%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:36:51.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:51 smithi042 ceph-mon[108570]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 703 op/s; 11543/368166 objects degraded (3.135%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:36:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:51.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:51 smithi159 ceph-mon[102519]: pgmap v286: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 703 op/s; 11543/368166 objects degraded (3.135%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:36:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:36:53.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:53 smithi042 ceph-mon[108570]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 86 B/s rd, 27 MiB/s wr, 754 op/s; 11543/362502 objects degraded (3.184%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:36:53.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:53 smithi042 ceph-mon[113926]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 86 B/s rd, 27 MiB/s wr, 754 op/s; 11543/362502 objects degraded (3.184%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:36:53.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:53 smithi159 ceph-mon[102519]: pgmap v287: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 86 B/s rd, 27 MiB/s wr, 754 op/s; 11543/362502 objects degraded (3.184%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:36:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:55 smithi042 ceph-mon[108570]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 796 op/s; 11355/355218 objects degraded (3.197%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:36:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 11355/355218 objects degraded (3.197%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:55 smithi042 ceph-mon[113926]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 796 op/s; 11355/355218 objects degraded (3.197%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:36:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 11355/355218 objects degraded (3.197%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:55.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:55 smithi159 ceph-mon[102519]: pgmap v288: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 17 MiB/s wr, 796 op/s; 11355/355218 objects degraded (3.197%); 4.0 MiB/s, 63 objects/s recovering 2024-09-13T14:36:55.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 11355/355218 objects degraded (3.197%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:36:57.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:57 smithi042 ceph-mon[108570]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.9 MiB/s wr, 642 op/s; 11355/355218 objects degraded (3.197%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:36:57.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:57 smithi042 ceph-mon[113926]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.9 MiB/s wr, 642 op/s; 11355/355218 objects degraded (3.197%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:36:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:57 smithi159 ceph-mon[102519]: pgmap v289: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 7.9 MiB/s wr, 642 op/s; 11355/355218 objects degraded (3.197%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:36:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:59 smithi042 ceph-mon[113926]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.8 MiB/s wr, 990 op/s; 11143/342699 objects degraded (3.252%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:36:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:59 smithi042 ceph-mon[108570]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.8 MiB/s wr, 990 op/s; 11143/342699 objects degraded (3.252%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:36:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:36:59.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:59 smithi159 ceph-mon[102519]: pgmap v290: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 7.8 MiB/s wr, 990 op/s; 11143/342699 objects degraded (3.252%); 3.8 MiB/s, 60 objects/s recovering 2024-09-13T14:36:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:36:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:36:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:36:59] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.51.0" 2024-09-13T14:37:00.562 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 11143/342699 objects degraded (3.252%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:00.562 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 11143/342699 objects degraded (3.252%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:00.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 11143/342699 objects degraded (3.252%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T14:37:01.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:01 smithi159 ceph-mon[102519]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 11143/342699 objects degraded (3.252%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:01.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:01 smithi042 ceph-mon[113926]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 11143/342699 objects degraded (3.252%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:01.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:01 smithi042 ceph-mon[108570]: pgmap v291: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 706 op/s; 11143/342699 objects degraded (3.252%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:01.882 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:37:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:02 smithi042 ceph-mon[113926]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:02 smithi042 ceph-mon[108570]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:02.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:02 smithi159 ceph-mon[102519]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (10m) 69s ago 17m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (9m) 70s ago 17m 80.1M - c8b91775d855 3a8b5443f86c 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (16m) 69s ago 16m 23.8M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (16m) 70s ago 16m 20.3M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (11m) 70s ago 21m 456M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (12m) 69s ago 22m 520M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (8m) 69s ago 23m 63.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:37:02.990 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (6m) 70s ago 21m 45.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (8m) 69s ago 21m 47.4M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (11m) 69s ago 16m 10.6M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (11m) 70s ago 16m 11.1M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (2m) 69s ago 20m 297M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (77s) 69s ago 20m 13.0M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (20m) 69s ago 20m 696M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (19m) 69s ago 19m 767M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (19m) 70s ago 19m 528M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (19m) 70s ago 19m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (18m) 70s ago 18m 594M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (18m) 70s ago 18m 475M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:37:02.991 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (10m) 70s ago 17m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:37:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[113926]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 11143/337407 objects degraded (3.303%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[113926]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[113926]: from='client.54292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[108570]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 11143/337407 objects degraded (3.303%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[108570]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:03 smithi042 ceph-mon[108570]: from='client.54292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:03.656 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:37:03.656 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:37:03.656 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:37:03.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:37:03.657 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:37:03.658 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:37:03.658 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:37:03.658 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:37:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:03 smithi159 ceph-mon[102519]: pgmap v292: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 853 op/s; 11143/337407 objects degraded (3.303%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:37:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:03 smithi159 ceph-mon[102519]: from='client.44351 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:03 smithi159 ceph-mon[102519]: from='client.54292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:04.234 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:37:04.234 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:37:04.234 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:37:04.234 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:37:04.234 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:37:04.235 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:37:04.235 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:37:04.235 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:37:04.235 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:37:04.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2865109609' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:04.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2865109609' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:04.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2865109609' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:05.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 10929/330957 objects degraded (3.302%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:05.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:05.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 10929/330957 objects degraded (3.302%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 10929/330957 objects degraded (3.302%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 10929/330957 objects degraded (3.302%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: pgmap v293: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 10929/330957 objects degraded (3.302%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: from='client.54304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:05.769 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 10929/330957 objects degraded (3.302%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:06.344 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:05 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:37:05.881+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:06.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:06 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:06 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:07 smithi159 ceph-mon[102519]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10929/330957 objects degraded (3.302%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:07 smithi042 ceph-mon[113926]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10929/330957 objects degraded (3.302%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:07 smithi042 ceph-mon[108570]: pgmap v294: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 10929/330957 objects degraded (3.302%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:09.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:09 smithi159 ceph-mon[102519]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 10714/320088 objects degraded (3.347%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:09] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.51.0" 2024-09-13T14:37:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:09 smithi042 ceph-mon[113926]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 10714/320088 objects degraded (3.347%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:09 smithi042 ceph-mon[108570]: pgmap v295: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 10714/320088 objects degraded (3.347%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:10.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 10714/320088 objects degraded (3.347%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 10714/320088 objects degraded (3.347%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 10714/320088 objects degraded (3.347%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:11.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:11 smithi159 ceph-mon[102519]: pgmap v296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10714/320088 objects degraded (3.347%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:11 smithi042 ceph-mon[113926]: pgmap v296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10714/320088 objects degraded (3.347%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:11 smithi042 ceph-mon[108570]: pgmap v296: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 627 op/s; 10714/320088 objects degraded (3.347%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:13.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:13 smithi159 ceph-mon[102519]: pgmap v297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 10714/314919 objects degraded (3.402%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:13 smithi042 ceph-mon[108570]: pgmap v297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 10714/314919 objects degraded (3.402%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:13 smithi042 ceph-mon[113926]: pgmap v297: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 10714/314919 objects degraded (3.402%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:15.900 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:15 smithi042 ceph-mon[113926]: pgmap v298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 10430/307989 objects degraded (3.386%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:37:15.900 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 10430/307989 objects degraded (3.386%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:15.901 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:15 smithi042 ceph-mon[108570]: pgmap v298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 10430/307989 objects degraded (3.386%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:37:15.901 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 10430/307989 objects degraded (3.386%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:15.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:15 smithi159 ceph-mon[102519]: pgmap v298: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 10430/307989 objects degraded (3.386%); 3.7 MiB/s, 59 objects/s recovering 2024-09-13T14:37:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 10430/307989 objects degraded (3.386%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:17.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:17 smithi159 ceph-mon[102519]: pgmap v299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10430/307989 objects degraded (3.386%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:37:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:17 smithi042 ceph-mon[108570]: pgmap v299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10430/307989 objects degraded (3.386%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:37:18.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:17 smithi042 ceph-mon[113926]: pgmap v299: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10430/307989 objects degraded (3.386%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:37:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:19 smithi159 ceph-mon[102519]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 10144/299055 objects degraded (3.392%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:37:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:19 smithi159 ceph-mon[102519]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:37:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:19 smithi042 ceph-mon[108570]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 10144/299055 objects degraded (3.392%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:37:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:19 smithi042 ceph-mon[108570]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:37:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:19 smithi042 ceph-mon[113926]: pgmap v300: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 10144/299055 objects degraded (3.392%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:37:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:19 smithi042 ceph-mon[113926]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T14:37:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:19] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.51.0" 2024-09-13T14:37:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:20 smithi159 ceph-mon[102519]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:37:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 10144/299055 objects degraded (3.392%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:20 smithi042 ceph-mon[113926]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:37:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 10144/299055 objects degraded (3.392%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:20 smithi042 ceph-mon[108570]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T14:37:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 10144/299055 objects degraded (3.392%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:21.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:20 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:37:20.898+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:21 smithi159 ceph-mon[102519]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 10144/295986 objects degraded (3.427%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:37:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:21 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:21 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[113926]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 10144/295986 objects degraded (3.427%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:37:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[108570]: pgmap v303: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 10144/295986 objects degraded (3.427%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:37:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:21 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:23 smithi159 ceph-mon[102519]: pgmap v304: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 9929/291648 objects degraded (3.404%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:37:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:23 smithi042 ceph-mon[113926]: pgmap v304: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 9929/291648 objects degraded (3.404%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:37:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:23 smithi042 ceph-mon[108570]: pgmap v304: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 9929/291648 objects degraded (3.404%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:37:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:25 smithi159 ceph-mon[102519]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 9890/285891 objects degraded (3.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:37:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 9890/285891 objects degraded (3.459%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:25 smithi042 ceph-mon[113926]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 9890/285891 objects degraded (3.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:37:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 9890/285891 objects degraded (3.459%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:25 smithi042 ceph-mon[108570]: pgmap v305: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 9890/285891 objects degraded (3.459%); 4.2 MiB/s, 67 objects/s recovering 2024-09-13T14:37:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 9890/285891 objects degraded (3.459%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:27.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:27 smithi159 ceph-mon[102519]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 9890/285891 objects degraded (3.459%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:27 smithi042 ceph-mon[113926]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 9890/285891 objects degraded (3.459%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:27 smithi042 ceph-mon[108570]: pgmap v306: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 548 op/s; 9890/285891 objects degraded (3.459%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:29.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:29 smithi159 ceph-mon[102519]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 9706/276402 objects degraded (3.512%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:37:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:29 smithi042 ceph-mon[113926]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 9706/276402 objects degraded (3.512%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:37:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:29 smithi042 ceph-mon[108570]: pgmap v307: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 9706/276402 objects degraded (3.512%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:37:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:29] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.51.0" 2024-09-13T14:37:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 9706/276402 objects degraded (3.512%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 9706/276402 objects degraded (3.512%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 9706/276402 objects degraded (3.512%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:31.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:31 smithi159 ceph-mon[102519]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 9706/273366 objects degraded (3.551%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:37:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:31 smithi042 ceph-mon[113926]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 9706/273366 objects degraded (3.551%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:37:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:31 smithi042 ceph-mon[108570]: pgmap v308: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 723 op/s; 9706/273366 objects degraded (3.551%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:37:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:37:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=cleanup t=2024-09-13T14:37:32.297021897Z level=info msg="Completed cleanup jobs" duration=3.188717ms 2024-09-13T14:37:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:37:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana.update.checker t=2024-09-13T14:37:32.469933322Z level=info msg="Update check succeeded" duration=66.473967ms 2024-09-13T14:37:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:37:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugins.update.checker t=2024-09-13T14:37:32.503853531Z level=info msg="Update check succeeded" duration=100.409478ms 2024-09-13T14:37:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:33 smithi159 ceph-mon[102519]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9509/268086 objects degraded (3.547%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:37:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:33 smithi042 ceph-mon[113926]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9509/268086 objects degraded (3.547%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:37:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:33 smithi042 ceph-mon[108570]: pgmap v309: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 9509/268086 objects degraded (3.547%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:37:34.826 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (10m) 30s ago 17m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (10m) 31s ago 17m 80.6M - c8b91775d855 3a8b5443f86c 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (16m) 30s ago 16m 24.3M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (16m) 31s ago 16m 20.6M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:37:35.913 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (12m) 31s ago 21m 457M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (12m) 30s ago 23m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (9m) 30s ago 23m 66.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (7m) 31s ago 22m 46.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (8m) 30s ago 22m 46.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (11m) 30s ago 17m 10.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (11m) 31s ago 17m 10.6M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (3m) 30s ago 21m 383M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (110s) 30s ago 21m 332M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (20m) 30s ago 20m 973M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (20m) 30s ago 20m 1060M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (19m) 31s ago 19m 581M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:37:35.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (19m) 31s ago 19m 728M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:37:35.915 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (19m) 31s ago 19m 657M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:37:35.915 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (18m) 31s ago 18m 521M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:37:35.915 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (11m) 31s ago 18m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:37:35.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:35 smithi159 ceph-mon[102519]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 9509/261228 objects degraded (3.640%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:35 smithi159 ceph-mon[102519]: from='client.54310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 9509/261228 objects degraded (3.640%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[113926]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 9509/261228 objects degraded (3.640%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[113926]: from='client.54310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 9509/261228 objects degraded (3.640%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[108570]: pgmap v310: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 9509/261228 objects degraded (3.640%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[108570]: from='client.54310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 9509/261228 objects degraded (3.640%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:36.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:37:35.916+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:36.583 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:37:36.583 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:37:36.583 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:37:36.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:36.583 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:37:36.584 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:37:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/496217353' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:36 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/496217353' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='client.54316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:36 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/496217353' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:37:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:37:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:37:37.155 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:37:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:37 smithi159 ceph-mon[102519]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9509/261228 objects degraded (3.640%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:37 smithi042 ceph-mon[113926]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9509/261228 objects degraded (3.640%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:37 smithi042 ceph-mon[108570]: pgmap v311: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 9509/261228 objects degraded (3.640%); 2.0 MiB/s, 31 objects/s recovering 2024-09-13T14:37:38.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:38 smithi159 ceph-mon[102519]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:38 smithi042 ceph-mon[113926]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:38 smithi042 ceph-mon[108570]: from='client.34434 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:37:39.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:39 smithi159 ceph-mon[102519]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 9319/252381 objects degraded (3.692%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:37:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:39 smithi042 ceph-mon[113926]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 9319/252381 objects degraded (3.692%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:37:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:39 smithi042 ceph-mon[108570]: pgmap v312: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 930 op/s; 9319/252381 objects degraded (3.692%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:37:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:39] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.51.0" 2024-09-13T14:37:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 9319/252381 objects degraded (3.692%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 9319/252381 objects degraded (3.692%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 9319/252381 objects degraded (3.692%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:41 smithi042 ceph-mon[113926]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9319/249558 objects degraded (3.734%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:37:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:41 smithi042 ceph-mon[108570]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9319/249558 objects degraded (3.734%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:37:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:41 smithi159 ceph-mon[102519]: pgmap v313: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 9319/249558 objects degraded (3.734%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T14:37:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:43 smithi042 ceph-mon[113926]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 9089/244329 objects degraded (3.720%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:43 smithi042 ceph-mon[108570]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 9089/244329 objects degraded (3.720%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:43 smithi159 ceph-mon[102519]: pgmap v314: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 9089/244329 objects degraded (3.720%); 3.2 MiB/s, 51 objects/s recovering 2024-09-13T14:37:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:45 smithi042 ceph-mon[113926]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 9089/237555 objects degraded (3.826%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:45 smithi042 ceph-mon[108570]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 9089/237555 objects degraded (3.826%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:45 smithi159 ceph-mon[102519]: pgmap v315: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 9089/237555 objects degraded (3.826%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:47 smithi042 ceph-mon[108570]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:47 smithi042 ceph-mon[113926]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:47 smithi159 ceph-mon[102519]: pgmap v316: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 9089/237555 objects degraded (3.826%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:37:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:49 smithi042 ceph-mon[113926]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 8871/228327 objects degraded (3.885%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:49 smithi042 ceph-mon[108570]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 8871/228327 objects degraded (3.885%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:49] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.51.0" 2024-09-13T14:37:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:49 smithi159 ceph-mon[102519]: pgmap v317: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 913 op/s; 8871/228327 objects degraded (3.885%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:37:51.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:50 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:37:50.932+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8871/228327 objects degraded (3.885%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8871/228327 objects degraded (3.885%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8871/228327 objects degraded (3.885%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:37:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[113926]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8871/225318 objects degraded (3.937%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T14:37:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[108570]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8871/225318 objects degraded (3.937%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T14:37:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:51 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:51 smithi159 ceph-mon[102519]: pgmap v318: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8871/225318 objects degraded (3.937%); 2.3 MiB/s, 37 objects/s recovering 2024-09-13T14:37:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:37:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:51 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:37:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:51 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:37:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:53 smithi042 ceph-mon[108570]: pgmap v319: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 8657/219903 objects degraded (3.937%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:37:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:53 smithi042 ceph-mon[113926]: pgmap v319: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 8657/219903 objects degraded (3.937%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:37:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:53 smithi159 ceph-mon[102519]: pgmap v319: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 8657/219903 objects degraded (3.937%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:37:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:55 smithi042 ceph-mon[108570]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8657/213066 objects degraded (4.063%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:55 smithi042 ceph-mon[113926]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8657/213066 objects degraded (4.063%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:55 smithi159 ceph-mon[102519]: pgmap v320: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8657/213066 objects degraded (4.063%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:37:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:57 smithi042 ceph-mon[113926]: pgmap v321: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:57 smithi042 ceph-mon[108570]: pgmap v321: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:37:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:57 smithi159 ceph-mon[102519]: pgmap v321: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 8657/213066 objects degraded (4.063%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:38:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:37:59 smithi042 ceph-mon[113926]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 8405/203778 objects degraded (4.125%); 3.6 MiB/s, 56 objects/s recovering 2024-09-13T14:38:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:37:59 smithi042 ceph-mon[108570]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 8405/203778 objects degraded (4.125%); 3.6 MiB/s, 56 objects/s recovering 2024-09-13T14:38:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:37:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:37:59] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.51.0" 2024-09-13T14:38:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:37:59 smithi159 ceph-mon[102519]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 8405/203778 objects degraded (4.125%); 3.6 MiB/s, 56 objects/s recovering 2024-09-13T14:38:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8405/203778 objects degraded (4.125%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8405/203778 objects degraded (4.125%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8405/203778 objects degraded (4.125%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:01 smithi042 ceph-mon[113926]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8405/200670 objects degraded (4.188%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:38:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:01 smithi042 ceph-mon[108570]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8405/200670 objects degraded (4.188%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:38:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:01 smithi159 ceph-mon[102519]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 8405/200670 objects degraded (4.188%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:38:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:03 smithi042 ceph-mon[108570]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 8135/195576 objects degraded (4.160%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:38:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:03 smithi042 ceph-mon[113926]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 8135/195576 objects degraded (4.160%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:38:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:03 smithi159 ceph-mon[102519]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 825 op/s; 8135/195576 objects degraded (4.160%); 3.8 MiB/s, 61 objects/s recovering 2024-09-13T14:38:05.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:04 smithi159 ceph-mon[102519]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:38:05.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:04 smithi042 ceph-mon[108570]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:38:05.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:04 smithi042 ceph-mon[113926]: osdmap e116: 8 total, 8 up, 8 in 2024-09-13T14:38:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:05 smithi159 ceph-mon[102519]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8135/188898 objects degraded (4.307%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:38:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:05 smithi159 ceph-mon[102519]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:38:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8135/188898 objects degraded (4.307%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[113926]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8135/188898 objects degraded (4.307%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:38:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[113926]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:38:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8135/188898 objects degraded (4.307%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[108570]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 8135/188898 objects degraded (4.307%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:38:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[108570]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T14:38:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8135/188898 objects degraded (4.307%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:07.031 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:07.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:07.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:07.672 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:38:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:07 smithi159 ceph-mon[102519]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 8135/188898 objects degraded (4.307%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:38:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:07 smithi042 ceph-mon[113926]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 8135/188898 objects degraded (4.307%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:38:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:07 smithi042 ceph-mon[108570]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 8135/188898 objects degraded (4.307%); 2.1 MiB/s, 33 objects/s recovering 2024-09-13T14:38:08.635 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:38:08.635 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (11m) 63s ago 18m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:38:08.635 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (10m) 64s ago 18m 80.6M - c8b91775d855 3a8b5443f86c 2024-09-13T14:38:08.635 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (17m) 63s ago 17m 24.3M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:38:08.635 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (17m) 64s ago 17m 20.6M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (12m) 64s ago 22m 457M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (13m) 63s ago 23m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (9m) 63s ago 24m 66.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (7m) 64s ago 22m 46.1M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (9m) 63s ago 22m 46.9M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (12m) 63s ago 17m 10.9M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (12m) 64s ago 17m 10.6M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (3m) 63s ago 22m 383M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (2m) 63s ago 21m 332M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:38:08.636 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (21m) 63s ago 21m 973M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (20m) 63s ago 20m 1060M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (20m) 64s ago 20m 581M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (20m) 64s ago 20m 728M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (19m) 64s ago 19m 657M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (19m) 64s ago 19m 521M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:38:08.637 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (11m) 64s ago 18m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:38:08.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:08 smithi159 ceph-mon[102519]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:08 smithi042 ceph-mon[108570]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:08 smithi042 ceph-mon[113926]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:09.232 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:38:09.232 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:38:09.232 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:38:09.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:09.232 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:38:09.233 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:38:09.772 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:38:10.015 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[113926]: pgmap v329: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 8011/181056 objects degraded (4.425%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:38:10.015 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[113926]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.015 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[113926]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2657395898' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:09] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.51.0" 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[108570]: pgmap v329: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 8011/181056 objects degraded (4.425%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[108570]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[108570]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.016 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2657395898' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:09 smithi159 ceph-mon[102519]: pgmap v329: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 816 op/s; 8011/181056 objects degraded (4.425%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:38:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:09 smithi159 ceph-mon[102519]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:09 smithi159 ceph-mon[102519]: from='client.44393 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2657395898' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:10 smithi159 ceph-mon[102519]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8011/181056 objects degraded (4.425%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:10 smithi042 ceph-mon[108570]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8011/181056 objects degraded (4.425%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:11.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:10 smithi042 ceph-mon[113926]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:11.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8011/181056 objects degraded (4.425%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:12.021 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:11 smithi159 ceph-mon[102519]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7767/178902 objects degraded (4.341%); 2.9 MiB/s, 45 objects/s recovering 2024-09-13T14:38:12.143 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:11 smithi042 ceph-mon[108570]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7767/178902 objects degraded (4.341%); 2.9 MiB/s, 45 objects/s recovering 2024-09-13T14:38:12.144 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:11 smithi042 ceph-mon[113926]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7767/178902 objects degraded (4.341%); 2.9 MiB/s, 45 objects/s recovering 2024-09-13T14:38:13.410 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[108570]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 7767/175047 objects degraded (4.437%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:13.411 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:13.411 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:13.411 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[113926]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 7767/175047 objects degraded (4.437%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:13.411 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:13.411 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:13 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:13.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:13 smithi159 ceph-mon[102519]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 559 op/s; 7767/175047 objects degraded (4.437%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:13 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:13 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:38:15.544+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:15.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7599/168774 objects degraded (4.502%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:38:15.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7599/168774 objects degraded (4.502%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:15.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7599/168774 objects degraded (4.502%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7599/168774 objects degraded (4.502%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 7599/168774 objects degraded (4.502%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7599/168774 objects degraded (4.502%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:16.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:16.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:16 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:16.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:16 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:16 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:16 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:16 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:16 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:17.927 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:17 smithi042 ceph-mon[113926]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7599/168774 objects degraded (4.502%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:17.928 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:17 smithi042 ceph-mon[108570]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7599/168774 objects degraded (4.502%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:17.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:17 smithi159 ceph-mon[102519]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 7599/168774 objects degraded (4.502%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:19.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:19 smithi159 ceph-mon[102519]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 7599/160443 objects degraded (4.736%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:19 smithi042 ceph-mon[113926]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 7599/160443 objects degraded (4.736%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:19 smithi042 ceph-mon[108570]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 7599/160443 objects degraded (4.736%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:38:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:19] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.51.0" 2024-09-13T14:38:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7599/160443 objects degraded (4.736%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7599/160443 objects degraded (4.736%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7599/160443 objects degraded (4.736%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:21.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:21 smithi159 ceph-mon[102519]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7442/157671 objects degraded (4.720%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:21 smithi042 ceph-mon[113926]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7442/157671 objects degraded (4.720%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:21 smithi042 ceph-mon[108570]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 649 op/s; 7442/157671 objects degraded (4.720%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:38:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:23 smithi159 ceph-mon[102519]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7442/153297 objects degraded (4.855%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:23 smithi042 ceph-mon[113926]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7442/153297 objects degraded (4.855%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:23 smithi042 ceph-mon[108570]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 7442/153297 objects degraded (4.855%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:25.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:25 smithi159 ceph-mon[102519]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7273/145731 objects degraded (4.991%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:38:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7273/145731 objects degraded (4.991%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:25 smithi042 ceph-mon[113926]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7273/145731 objects degraded (4.991%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:38:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7273/145731 objects degraded (4.991%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:25 smithi042 ceph-mon[108570]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 7273/145731 objects degraded (4.991%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:38:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7273/145731 objects degraded (4.991%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:27.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:27 smithi159 ceph-mon[102519]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7273/145731 objects degraded (4.991%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:28.057 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:27 smithi042 ceph-mon[113926]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7273/145731 objects degraded (4.991%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:28.057 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:27 smithi042 ceph-mon[108570]: pgmap v338: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 7273/145731 objects degraded (4.991%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:29.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:29 smithi159 ceph-mon[102519]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7273/136377 objects degraded (5.333%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:29 smithi042 ceph-mon[113926]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7273/136377 objects degraded (5.333%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:29 smithi042 ceph-mon[108570]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 7273/136377 objects degraded (5.333%); 1.7 MiB/s, 27 objects/s recovering 2024-09-13T14:38:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:29] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.51.0" 2024-09-13T14:38:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7273/136377 objects degraded (5.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7273/136377 objects degraded (5.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:30.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:30.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:30 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:38:30.568+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7273/136377 objects degraded (5.333%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:30.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:31.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:31 smithi159 ceph-mon[102519]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 7083/133290 objects degraded (5.314%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:38:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:31 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:31 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[113926]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 7083/133290 objects degraded (5.314%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:38:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[108570]: pgmap v340: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 7083/133290 objects degraded (5.314%); 2.7 MiB/s, 42 objects/s recovering 2024-09-13T14:38:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:31 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:33 smithi159 ceph-mon[102519]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7083/128721 objects degraded (5.503%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T14:38:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:33 smithi042 ceph-mon[108570]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7083/128721 objects degraded (5.503%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T14:38:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:33 smithi042 ceph-mon[113926]: pgmap v341: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 7083/128721 objects degraded (5.503%); 1.9 MiB/s, 29 objects/s recovering 2024-09-13T14:38:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:35 smithi159 ceph-mon[102519]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 6858/121080 objects degraded (5.664%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:38:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6858/121080 objects degraded (5.664%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:35 smithi042 ceph-mon[113926]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 6858/121080 objects degraded (5.664%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:38:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6858/121080 objects degraded (5.664%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:35 smithi042 ceph-mon[108570]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 894 op/s; 6858/121080 objects degraded (5.664%); 3.0 MiB/s, 48 objects/s recovering 2024-09-13T14:38:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6858/121080 objects degraded (5.664%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:36.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:37.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:37 smithi159 ceph-mon[102519]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6858/121080 objects degraded (5.664%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:38.192 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:37 smithi042 ceph-mon[113926]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6858/121080 objects degraded (5.664%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:38.193 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:37 smithi042 ceph-mon[108570]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6858/121080 objects degraded (5.664%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:39 smithi159 ceph-mon[102519]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 6858/111900 objects degraded (6.129%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:39 smithi042 ceph-mon[113926]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 6858/111900 objects degraded (6.129%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:39 smithi042 ceph-mon[108570]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 939 op/s; 6858/111900 objects degraded (6.129%); 2.2 MiB/s, 34 objects/s recovering 2024-09-13T14:38:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:39] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-09-13T14:38:40.367 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:38:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6858/111900 objects degraded (6.129%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6858/111900 objects degraded (6.129%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6858/111900 objects degraded (6.129%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (11m) 26s ago 18m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (11m) 29s ago 18m 83.2M - c8b91775d855 3a8b5443f86c 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (17m) 26s ago 17m 24.7M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (17m) 29s ago 17m 20.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (13m) 29s ago 22m 457M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:38:41.493 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (14m) 26s ago 24m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (10m) 26s ago 24m 67.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (8m) 29s ago 23m 47.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (9m) 26s ago 23m 49.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (12m) 26s ago 18m 11.6M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (12m) 29s ago 18m 9.83M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (4m) 26s ago 22m 452M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (2m) 26s ago 22m 370M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (21m) 26s ago 21m 986M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (21m) 26s ago 21m 1081M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (21m) 29s ago 21m 586M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (20m) 29s ago 20m 969M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:38:41.494 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (20m) 29s ago 20m 672M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:38:41.495 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (19m) 29s ago 19m 534M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:38:41.495 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (12m) 29s ago 19m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:38:41.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[113926]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6627/108789 objects degraded (6.092%); 2.5 MiB/s, 53 objects/s recovering 2024-09-13T14:38:41.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[113926]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[113926]: from='client.54358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[108570]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6627/108789 objects degraded (6.092%); 2.5 MiB/s, 53 objects/s recovering 2024-09-13T14:38:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[108570]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:41 smithi042 ceph-mon[108570]: from='client.54358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:41 smithi159 ceph-mon[102519]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 765 op/s; 6627/108789 objects degraded (6.092%); 2.5 MiB/s, 53 objects/s recovering 2024-09-13T14:38:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:41 smithi159 ceph-mon[102519]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:41 smithi159 ceph-mon[102519]: from='client.54358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:42.163 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:38:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:38:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:38:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:38:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:38:42.165 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:38:42.744 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:38:42.745 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:38:42.745 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:38:42.745 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:38:43.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:42 smithi042 ceph-mon[113926]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1017676570' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:43.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:42 smithi042 ceph-mon[108570]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1017676570' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:43.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:42 smithi159 ceph-mon[102519]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1017676570' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:38:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:43 smithi042 ceph-mon[108570]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 6627/104358 objects degraded (6.350%); 1.5 MiB/s, 37 objects/s recovering 2024-09-13T14:38:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:43 smithi042 ceph-mon[108570]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:43 smithi042 ceph-mon[113926]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 6627/104358 objects degraded (6.350%); 1.5 MiB/s, 37 objects/s recovering 2024-09-13T14:38:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:43 smithi042 ceph-mon[113926]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:43 smithi159 ceph-mon[102519]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 6627/104358 objects degraded (6.350%); 1.5 MiB/s, 37 objects/s recovering 2024-09-13T14:38:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:43 smithi159 ceph-mon[102519]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:38:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[108570]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 6401/97236 objects degraded (6.583%); 1.5 MiB/s, 56 objects/s recovering 2024-09-13T14:38:45.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6401/97236 objects degraded (6.583%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:45.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[113926]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 6401/97236 objects degraded (6.583%); 1.5 MiB/s, 56 objects/s recovering 2024-09-13T14:38:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6401/97236 objects degraded (6.583%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:45.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:45.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:45 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:38:45.586+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:45 smithi159 ceph-mon[102519]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 6401/97236 objects degraded (6.583%); 1.5 MiB/s, 56 objects/s recovering 2024-09-13T14:38:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6401/97236 objects degraded (6.583%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:38:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:46 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:46 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:46 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:46 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:47.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:46 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:38:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:46 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T14:38:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:47 smithi042 ceph-mon[113926]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6401/97236 objects degraded (6.583%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:47 smithi042 ceph-mon[108570]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6401/97236 objects degraded (6.583%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:47 smithi159 ceph-mon[102519]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 6401/97236 objects degraded (6.583%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:49 smithi042 ceph-mon[113926]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 6401/88092 objects degraded (7.266%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:49 smithi042 ceph-mon[108570]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 6401/88092 objects degraded (7.266%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:49] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-09-13T14:38:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:49 smithi159 ceph-mon[102519]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.8 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 915 op/s; 6401/88092 objects degraded (7.266%); 336 KiB/s, 38 objects/s recovering 2024-09-13T14:38:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6401/88092 objects degraded (7.266%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6401/88092 objects degraded (7.266%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6401/88092 objects degraded (7.266%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:38:52.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:51 smithi042 ceph-mon[113926]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6144/85026 objects degraded (7.226%); 336 KiB/s, 59 objects/s recovering 2024-09-13T14:38:52.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:51 smithi042 ceph-mon[108570]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6144/85026 objects degraded (7.226%); 336 KiB/s, 59 objects/s recovering 2024-09-13T14:38:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:51 smithi159 ceph-mon[102519]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6144/85026 objects degraded (7.226%); 336 KiB/s, 59 objects/s recovering 2024-09-13T14:38:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:53 smithi042 ceph-mon[108570]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6144/80463 objects degraded (7.636%); 0 B/s, 40 objects/s recovering 2024-09-13T14:38:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:53 smithi042 ceph-mon[108570]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:38:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:53 smithi042 ceph-mon[113926]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6144/80463 objects degraded (7.636%); 0 B/s, 40 objects/s recovering 2024-09-13T14:38:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:53 smithi042 ceph-mon[113926]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:38:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:53 smithi159 ceph-mon[102519]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 6144/80463 objects degraded (7.636%); 0 B/s, 40 objects/s recovering 2024-09-13T14:38:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:53 smithi159 ceph-mon[102519]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T14:38:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:54 smithi042 ceph-mon[108570]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:38:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:54 smithi042 ceph-mon[113926]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:38:55.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:54 smithi159 ceph-mon[102519]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T14:38:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:55 smithi042 ceph-mon[108570]: pgmap v354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:38:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6141/73785 objects degraded (8.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:55 smithi042 ceph-mon[113926]: pgmap v354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:38:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6141/73785 objects degraded (8.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:55 smithi159 ceph-mon[102519]: pgmap v354: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 976 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:38:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6141/73785 objects degraded (8.323%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-09-13T14:38:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:57 smithi042 ceph-mon[108570]: pgmap v355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:38:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:57 smithi042 ceph-mon[113926]: pgmap v355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:38:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:57 smithi159 ceph-mon[102519]: pgmap v355: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 595 op/s; 6141/73785 objects degraded (8.323%); 0 B/s, 32 objects/s recovering 2024-09-13T14:39:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:38:59 smithi042 ceph-mon[113926]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 5618/64575 objects degraded (8.700%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:38:59 smithi042 ceph-mon[108570]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 5618/64575 objects degraded (8.700%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:38:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:38:59] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.51.0" 2024-09-13T14:39:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:38:59 smithi159 ceph-mon[102519]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 5618/64575 objects degraded (8.700%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5618/64575 objects degraded (8.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:00 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5618/64575 objects degraded (8.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:00 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:01.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:00 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:39:00.602+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5618/64575 objects degraded (8.700%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:00 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[108570]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5618/62490 objects degraded (8.990%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[113926]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5618/62490 objects degraded (8.990%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:02.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:01 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:01 smithi159 ceph-mon[102519]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5618/62490 objects degraded (8.990%); 0 B/s, 65 objects/s recovering 2024-09-13T14:39:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:01 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:01 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:03 smithi042 ceph-mon[108570]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5618/57735 objects degraded (9.731%); 0 B/s, 56 objects/s recovering 2024-09-13T14:39:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:03 smithi042 ceph-mon[113926]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5618/57735 objects degraded (9.731%); 0 B/s, 56 objects/s recovering 2024-09-13T14:39:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:03 smithi159 ceph-mon[102519]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.2 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 647 op/s; 5618/57735 objects degraded (9.731%); 0 B/s, 56 objects/s recovering 2024-09-13T14:39:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[108570]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 67 objects/s recovering 2024-09-13T14:39:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5441/49842 objects degraded (10.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[113926]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 67 objects/s recovering 2024-09-13T14:39:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5441/49842 objects degraded (10.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:05 smithi159 ceph-mon[102519]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 67 objects/s recovering 2024-09-13T14:39:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5441/49842 objects degraded (10.916%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:07 smithi042 ceph-mon[108570]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 58 objects/s recovering 2024-09-13T14:39:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:07 smithi042 ceph-mon[113926]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 58 objects/s recovering 2024-09-13T14:39:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:07 smithi159 ceph-mon[102519]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 1.0 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 5441/49842 objects degraded (10.916%); 0 B/s, 58 objects/s recovering 2024-09-13T14:39:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:09 smithi042 ceph-mon[113926]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 818 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 5273/39297 objects degraded (13.418%); 0 B/s, 72 objects/s recovering 2024-09-13T14:39:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:09 smithi042 ceph-mon[108570]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 818 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 5273/39297 objects degraded (13.418%); 0 B/s, 72 objects/s recovering 2024-09-13T14:39:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:09] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-13T14:39:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:09 smithi159 ceph-mon[102519]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 818 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 957 op/s; 5273/39297 objects degraded (13.418%); 0 B/s, 72 objects/s recovering 2024-09-13T14:39:11.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5273/39297 objects degraded (13.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5273/39297 objects degraded (13.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5273/39297 objects degraded (13.418%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:11 smithi159 ceph-mon[102519]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 769 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 5273/36984 objects degraded (14.258%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:11 smithi042 ceph-mon[108570]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 769 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 5273/36984 objects degraded (14.258%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:11 smithi042 ceph-mon[113926]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 769 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 5273/36984 objects degraded (14.258%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:13.351 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:39:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:13 smithi159 ceph-mon[102519]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 669 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 5273/32181 objects degraded (16.385%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:14.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:13 smithi042 ceph-mon[108570]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 669 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 5273/32181 objects degraded (16.385%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:14.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:13 smithi042 ceph-mon[113926]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 669 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 841 op/s; 5273/32181 objects degraded (16.385%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (12m) 59s ago 19m 18.6M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (11m) 62s ago 19m 83.2M - c8b91775d855 3a8b5443f86c 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (18m) 59s ago 18m 24.7M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (18m) 62s ago 18m 20.9M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (13m) 62s ago 23m 457M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:39:14.465 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (14m) 59s ago 25m 521M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (11m) 59s ago 25m 67.8M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (8m) 62s ago 23m 47.5M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (10m) 59s ago 23m 49.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (13m) 59s ago 19m 11.6M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (13m) 62s ago 18m 9.83M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (4m) 59s ago 23m 452M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (3m) 59s ago 22m 370M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (22m) 59s ago 22m 986M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (21m) 59s ago 21m 1081M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (21m) 62s ago 21m 586M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (21m) 62s ago 21m 969M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:39:14.466 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (20m) 62s ago 20m 672M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:39:14.467 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (20m) 62s ago 20m 534M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:39:14.467 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (12m) 62s ago 19m 47.6M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:39:15.140 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:14 smithi042 ceph-mon[108570]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.140 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:14 smithi042 ceph-mon[108570]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.141 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:14 smithi042 ceph-mon[113926]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.141 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:14 smithi042 ceph-mon[113926]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.141 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:39:15.141 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:39:15.141 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:39:15.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:15.141 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 6, 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 8, 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:39:15.142 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:39:15.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:14 smithi159 ceph-mon[102519]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:14 smithi159 ceph-mon[102519]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:15.702 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:39:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:39:15.703 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:39:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[108570]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 43 objects/s recovering 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[108570]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1275327364' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5101/24306 objects degraded (20.987%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[113926]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 43 objects/s recovering 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[113926]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1275327364' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:16.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5101/24306 objects degraded (20.987%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:16.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:16.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:15 smithi159 ceph-mon[102519]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 928 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 43 objects/s recovering 2024-09-13T14:39:16.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:15 smithi159 ceph-mon[102519]: from='client.54379 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:16.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1275327364' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:16.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5101/24306 objects degraded (20.987%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:16.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:16.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:16 smithi159 ceph-mon[102519]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:17.216 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:16 smithi042 ceph-mon[108570]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:17.217 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:16 smithi042 ceph-mon[113926]: from='client.44432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:17.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:17 smithi159 ceph-mon[102519]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:17 smithi042 ceph-mon[108570]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:17 smithi042 ceph-mon[113926]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 505 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 5101/24306 objects degraded (20.987%); 0 B/s, 28 objects/s recovering 2024-09-13T14:39:19.918 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:19] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-13T14:39:20.174 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:19 smithi159 ceph-mon[102519]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 4944/15162 objects degraded (32.608%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:20.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:19 smithi042 ceph-mon[108570]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 4944/15162 objects degraded (32.608%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:19 smithi042 ceph-mon[113926]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 4944/15162 objects degraded (32.608%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4944/15162 objects degraded (32.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4944/15162 objects degraded (32.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4944/15162 objects degraded (32.608%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:21.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:21 smithi159 ceph-mon[102519]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 270 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4944/13026 objects degraded (37.955%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[113926]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 270 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4944/13026 objects degraded (37.955%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[108570]: pgmap v367: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 270 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 729 op/s; 4944/13026 objects degraded (37.955%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:22.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:22.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:22.881 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 124586 objects 2024-09-13T14:39:22.881 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :158.117 2024-09-13T14:39:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:39:22.866+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:23.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:23.844 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-13T14:39:23.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-13T14:39:24.294 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[108570]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 180 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 4944/8685 objects degraded (56.926%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[113926]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 180 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 4944/8685 objects degraded (56.926%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:24 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:24.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:24 smithi159 ceph-mon[102519]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 180 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 4944/8685 objects degraded (56.926%); 0 B/s, 27 objects/s recovering 2024-09-13T14:39:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:24 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:24 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T14:39:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:26 smithi042 ceph-mon[108570]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4777/1587 objects degraded (301.008%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:26 smithi042 ceph-mon[113926]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4777/1587 objects degraded (301.008%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:26.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:26 smithi159 ceph-mon[102519]: pgmap v369: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 41 objects/s recovering 2024-09-13T14:39:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4777/1587 objects degraded (301.008%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:27.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:27 smithi042 ceph-mon[113926]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:39:27.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:27 smithi042 ceph-mon[113926]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 32 objects/s recovering 2024-09-13T14:39:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:27 smithi042 ceph-mon[108570]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:39:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:27 smithi042 ceph-mon[108570]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 32 objects/s recovering 2024-09-13T14:39:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:27 smithi159 ceph-mon[102519]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T14:39:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:27 smithi159 ceph-mon[102519]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 32 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 4777/1587 objects degraded (301.008%); 0 B/s, 32 objects/s recovering 2024-09-13T14:39:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:28 smithi042 ceph-mon[108570]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:39:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:28 smithi042 ceph-mon[113926]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:39:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:28 smithi159 ceph-mon[102519]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T14:39:29.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:29 smithi042 ceph-mon[108570]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:29.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:29 smithi042 ceph-mon[113926]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:29.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:29 smithi159 ceph-mon[102519]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 539 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:29] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-09-13T14:39:31.457 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:31 smithi042 ceph-mon[108570]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:31.457 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:31.458 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:31 smithi042 ceph-mon[113926]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:31.458 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:31 smithi159 ceph-mon[102519]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 358 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 214 objects/s recovering 2024-09-13T14:39:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:39:32.192 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_3' removed 2024-09-13T14:39:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3999431273' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:32 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.558 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3999431273' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.559 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:32 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.559 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3999431273' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.559 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:32 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:39:32.871 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T14:39:32.871 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-13T14:39:32.872 DEBUG:teuthology.parallel:result is None 2024-09-13T14:39:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:33 smithi159 ceph-mon[102519]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 193 objects/s recovering 2024-09-13T14:39:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:33 smithi159 ceph-mon[102519]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:39:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[108570]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 193 objects/s recovering 2024-09-13T14:39:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[108570]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:39:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[113926]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 62 op/s; 3226/78 objects degraded (4135.897%); 0 B/s, 193 objects/s recovering 2024-09-13T14:39:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:39:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:33 smithi042 ceph-mon[113926]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T14:39:34.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:34 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized) 2024-09-13T14:39:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:34 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:39:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:34 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized) 2024-09-13T14:39:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:34 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:39:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:34 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3226/78 objects degraded (4135.897%), 2 pgs degraded, 2 pgs undersized) 2024-09-13T14:39:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:34 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:39:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:35 smithi159 ceph-mon[102519]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:35 smithi042 ceph-mon[108570]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:35 smithi042 ceph-mon[113926]: pgmap v377: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:37 smithi042 ceph-mon[108570]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:37 smithi042 ceph-mon[113926]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:37 smithi159 ceph-mon[102519]: pgmap v378: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:38.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:38.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:38.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[113926]: Upgrade: osd.2 is safe to restart 2024-09-13T14:39:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:38.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:38 smithi042 ceph-mon[108570]: Upgrade: osd.2 is safe to restart 2024-09-13T14:39:39.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:38 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T14:39:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:38 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T14:39:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:38 smithi159 ceph-mon[102519]: Upgrade: osd.2 is safe to restart 2024-09-13T14:39:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[108570]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:39:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:39.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:39] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.51.0" 2024-09-13T14:39:39.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[113926]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:39.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:39.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:39:39.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:40.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:39 smithi159 ceph-mon[102519]: pgmap v379: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:40.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:40.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T14:39:40.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:40 smithi159 ceph-mon[102519]: Upgrade: Updating osd.2 2024-09-13T14:39:40.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:40 smithi159 ceph-mon[102519]: Deploying daemon osd.2 on smithi042 2024-09-13T14:39:41.040 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:40 smithi042 ceph-mon[113926]: Upgrade: Updating osd.2 2024-09-13T14:39:41.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:40 smithi042 ceph-mon[113926]: Deploying daemon osd.2 on smithi042 2024-09-13T14:39:41.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:40 smithi042 ceph-mon[108570]: Upgrade: Updating osd.2 2024-09-13T14:39:41.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:40 smithi042 ceph-mon[108570]: Deploying daemon osd.2 on smithi042 2024-09-13T14:39:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:41 smithi042 ceph-mon[113926]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:41 smithi042 ceph-mon[108570]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:41 smithi159 ceph-mon[102519]: pgmap v380: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:43.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:43 smithi042 ceph-mon[108570]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:43.932 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:43 smithi042 ceph-mon[113926]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:43.932 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:43 smithi042 systemd[1]: Stopping Ceph osd.2 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:39:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:43 smithi159 ceph-mon[102519]: pgmap v381: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:44.348 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[56893]: 2024-09-13T14:39:43.929+0000 7f1a5d401640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:39:44.348 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[56893]: 2024-09-13T14:39:43.929+0000 7f1a5d401640 -1 osd.2 122 *** Got signal Terminated *** 2024-09-13T14:39:44.348 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:43 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[56893]: 2024-09-13T14:39:43.929+0000 7f1a5d401640 -1 osd.2 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:39:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:44 smithi159 ceph-mon[102519]: osd.2 marked itself down and dead 2024-09-13T14:39:45.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:44 smithi042 ceph-mon[113926]: osd.2 marked itself down and dead 2024-09-13T14:39:45.250 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:44 smithi042 ceph-mon[108570]: osd.2 marked itself down and dead 2024-09-13T14:39:45.796 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:45 smithi042 podman[139109]: 2024-09-13 14:39:45.512262658 +0000 UTC m=+1.695651606 container died a72e8e212a9f7544be752c0e1418daa6a4a2ac20d64fe134ad9aa4a8a3ad09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[108570]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[108570]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[108570]: osdmap e123: 8 total, 7 up, 8 in 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[113926]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[113926]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:46.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:45 smithi042 ceph-mon[113926]: osdmap e123: 8 total, 7 up, 8 in 2024-09-13T14:39:46.074 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:45 smithi042 podman[139109]: 2024-09-13 14:39:45.88610962 +0000 UTC m=+2.069498558 container cleanup a72e8e212a9f7544be752c0e1418daa6a4a2ac20d64fe134ad9aa4a8a3ad09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:39:46.074 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:45 smithi042 bash[139109]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2 2024-09-13T14:39:46.074 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:39:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:45 smithi159 ceph-mon[102519]: pgmap v382: 73 pgs: 73 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:45 smithi159 ceph-mon[102519]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:39:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:45 smithi159 ceph-mon[102519]: osdmap e123: 8 total, 7 up, 8 in 2024-09-13T14:39:46.348 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139131]: 2024-09-13 14:39:46.187941912 +0000 UTC m=+0.667744229 container remove a72e8e212a9f7544be752c0e1418daa6a4a2ac20d64fe134ad9aa4a8a3ad09d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T14:39:46.744 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:39:46.744 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (12m) 25s ago 20m 18.3M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:39:46.744 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (12m) 24s ago 19m 78.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:39:46.744 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (18m) 25s ago 18m 25.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (18m) 24s ago 18m 21.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (14m) 24s ago 23m 458M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (15m) 25s ago 25m 522M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (11m) 25s ago 25m 68.6M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (9m) 24s ago 24m 47.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (11m) 25s ago 24m 49.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (13m) 25s ago 19m 10.8M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (13m) 24s ago 19m 9.92M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (5m) 25s ago 23m 664M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (4m) 25s ago 23m 394M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:39:46.745 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (22m) 25s ago 22m 991M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e a72e8e212a9f 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (22m) 25s ago 22m 1257M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (22m) 24s ago 22m 596M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (21m) 24s ago 21m 976M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (21m) 24s ago 21m 670M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (20m) 24s ago 20m 543M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:39:46.746 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (13m) 24s ago 20m 49.8M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:39:46.839 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139277]: 2024-09-13 14:39:46.487525867 +0000 UTC m=+0.021082957 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:46.840 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139277]: 2024-09-13 14:39:46.622403303 +0000 UTC m=+0.155960383 container create e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T14:39:46.840 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139277]: 2024-09-13 14:39:46.833257109 +0000 UTC m=+0.366814192 container init e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T14:39:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:46 smithi042 ceph-mon[113926]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T14:39:47.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:46 smithi042 ceph-mon[113926]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:46 smithi042 ceph-mon[108570]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T14:39:47.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:46 smithi042 ceph-mon[108570]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:47.099 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139277]: 2024-09-13 14:39:46.83852657 +0000 UTC m=+0.372083651 container start e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T14:39:47.099 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:46 smithi042 podman[139277]: 2024-09-13 14:39:46.958777864 +0000 UTC m=+0.492334954 container attach e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:39:47.125 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 5, 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 7, 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 7 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:39:47.126 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:39:47.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:46 smithi159 ceph-mon[102519]: osdmap e124: 8 total, 7 up, 8 in 2024-09-13T14:39:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:46 smithi159 ceph-mon[102519]: from='client.34524 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:47.430 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:39:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:39:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:39:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-13T14:39:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:39:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/8 daemons upgraded", 2024-09-13T14:39:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:39:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:39:47.431 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:39:47.431 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:47 smithi042 podman[139277]: 2024-09-13 14:39:47.120765493 +0000 UTC m=+0.654322574 container died e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[108570]: pgmap v385: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[108570]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[108570]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2177977564' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[113926]: pgmap v385: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[113926]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[113926]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2177977564' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:48.100 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:47 smithi042 podman[139363]: 2024-09-13 14:39:47.796286389 +0000 UTC m=+0.669226746 container remove e9b16aa4544b74320f230be61c355f824925a9c7e650d8871b9746c0b4a75be4 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T14:39:48.100 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:47 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T14:39:48.100 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:47 smithi042 systemd[1]: Stopped Ceph osd.2 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:39:48.100 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:47 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.2.service: Consumed 27min 41.035s CPU time. 2024-09-13T14:39:48.100 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 systemd[1]: Starting Ceph osd.2 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:39:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:47 smithi159 ceph-mon[102519]: pgmap v385: 73 pgs: 2 stale+active+clean, 71 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:39:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:47 smithi159 ceph-mon[102519]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:47 smithi159 ceph-mon[102519]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2177977564' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:48.592 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 podman[139495]: 2024-09-13 14:39:48.174099086 +0000 UTC m=+0.016506473 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:48.592 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 podman[139495]: 2024-09-13 14:39:48.280792009 +0000 UTC m=+0.123199395 container create 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T14:39:48.592 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 podman[139495]: 2024-09-13 14:39:48.508384817 +0000 UTC m=+0.350792204 container init 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True) 2024-09-13T14:39:48.592 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 podman[139495]: 2024-09-13 14:39:48.513590919 +0000 UTC m=+0.355998308 container start 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:39:48.592 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 podman[139495]: 2024-09-13 14:39:48.591683153 +0000 UTC m=+0.434090555 container attach 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T14:39:48.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:39:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:48 smithi042 bash[139495]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-13T14:39:49.239 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-mon[108570]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:49.239 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:48 smithi042 ceph-mon[113926]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:48 smithi159 ceph-mon[102519]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:39:49.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:39:49.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 bash[139495]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:39:49.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 bash[139495]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 bash[139495]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 bash[139495]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate[139510]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 bash[139495]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-13T14:39:49.599 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 podman[139495]: 2024-09-13 14:39:49.282795029 +0000 UTC m=+1.125202415 container died 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:39:49.996 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-mon[108570]: pgmap v386: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-09-13T14:39:49.997 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:49.997 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-mon[113926]: pgmap v386: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-09-13T14:39:49.997 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:49.998 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:49] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.51.0" 2024-09-13T14:39:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:49 smithi159 ceph-mon[102519]: pgmap v386: 73 pgs: 15 active+undersized, 2 active+undersized+degraded, 56 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 3/78 objects degraded (3.846%) 2024-09-13T14:39:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:49 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 3/78 objects degraded (3.846%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:50.264 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:49 smithi042 podman[139651]: 2024-09-13 14:39:49.995813094 +0000 UTC m=+0.703923069 container remove 3f1737e439a1323df1ca423121ac20ee7cec33afcbdf5c80655e55031a4bbd28 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:39:50.264 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 podman[139716]: 2024-09-13 14:39:50.116212701 +0000 UTC m=+0.017249152 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:39:50.264 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 podman[139716]: 2024-09-13 14:39:50.263658788 +0000 UTC m=+0.164695236 container create 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:39:50.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 podman[139716]: 2024-09-13 14:39:50.491391842 +0000 UTC m=+0.392428300 container init 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:39:50.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 podman[139716]: 2024-09-13 14:39:50.496748577 +0000 UTC m=+0.397785025 container start 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:39:51.098 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 ceph-osd[139754]: -- 172.21.15.42:0/779640745 <== mon.2 v2:172.21.15.42:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b36947a40 con 0x555b37714400 2024-09-13T14:39:51.098 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 bash[139716]: 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a 2024-09-13T14:39:51.098 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:50 smithi042 systemd[1]: Started Ceph osd.2 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: pgmap v387: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:51.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: pgmap v387: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: pgmap v387: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:39:52.298 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:39:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T14:39:51.991+0000 7f844e9f0640 -1 Falling back to public interface 2024-09-13T14:39:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:53 smithi159 ceph-mon[102519]: pgmap v388: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:53 smithi042 ceph-mon[108570]: pgmap v388: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:53 smithi042 ceph-mon[113926]: pgmap v388: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:55.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[113926]: pgmap v389: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:55.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[108570]: pgmap v389: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:55.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.446 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:55 smithi159 ceph-mon[102519]: pgmap v389: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:56.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:39:57.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:57 smithi159 ceph-mon[102519]: pgmap v390: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:57.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:57 smithi042 ceph-mon[108570]: pgmap v390: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:57.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:57 smithi042 ceph-mon[113926]: pgmap v390: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:58.598 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T14:39:58.482+0000 7f23e3861640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:39:59.727 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: pgmap v391: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:59.727 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.727 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[108570]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: pgmap v391: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:59.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:59.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-mon[113926]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: pgmap v391: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:39:59.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:39:59.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:39:59.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:39:59 smithi159 ceph-mon[102519]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T14:40:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:39:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:39:59] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.51.0" 2024-09-13T14:40:00.381 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T14:40:00.117+0000 7f844e9f0640 -1 osd.2 122 log_to_monitors true 2024-09-13T14:40:00.381 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T14:40:00.380+0000 7f8442792640 -1 osd.2 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: osd.2 (root=default,host=smithi042) is down 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: [WRN] PG_DEGRADED: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:00.382 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[108570]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: osd.2 (root=default,host=smithi042) is down 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: [WRN] PG_DEGRADED: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:00.383 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:00 smithi042 ceph-mon[113926]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: osd.2 (root=default,host=smithi042) is down 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: [WRN] PG_DEGRADED: Degraded data redundancy: 5/78 objects degraded (6.410%), 3 pgs degraded 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T14:40:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:00.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:00 smithi159 ceph-mon[102519]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T14:40:01.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:01 smithi159 ceph-mon[102519]: pgmap v392: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:40:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:01 smithi159 ceph-mon[102519]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:40:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:01 smithi159 ceph-mon[102519]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T14:40:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:01 smithi159 ceph-mon[102519]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:01 smithi159 ceph-mon[102519]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[108570]: pgmap v392: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[108570]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[108570]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[108570]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[108570]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[113926]: pgmap v392: 73 pgs: 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5/78 objects degraded (6.410%) 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[113926]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[113926]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[113926]: from='osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:01.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:01 smithi042 ceph-mon[113926]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:02.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:02 smithi159 ceph-mon[102519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:02 smithi159 ceph-mon[102519]: osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758] boot 2024-09-13T14:40:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:02 smithi159 ceph-mon[102519]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:40:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:02 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:40:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:02 smithi159 ceph-mon[102519]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:40:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[108570]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[108570]: osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758] boot 2024-09-13T14:40:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[108570]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:40:02.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[108570]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[113926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[113926]: osd.2 [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758] boot 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[113926]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T14:40:02.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:02 smithi042 ceph-mon[113926]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T14:40:03.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:03 smithi159 ceph-mon[102519]: pgmap v395: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:40:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:03 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:03 smithi042 ceph-mon[108570]: pgmap v395: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:40:03.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:03 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:03 smithi042 ceph-mon[113926]: pgmap v395: 73 pgs: 2 peering, 15 active+undersized, 2 active+undersized+degraded, 54 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4/78 objects degraded (5.128%) 2024-09-13T14:40:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:03 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4/78 objects degraded (5.128%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:05 smithi159 ceph-mon[102519]: pgmap v397: 73 pgs: 2 peering, 7 active+undersized, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:05.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:05 smithi042 ceph-mon[108570]: pgmap v397: 73 pgs: 2 peering, 7 active+undersized, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:05 smithi042 ceph-mon[113926]: pgmap v397: 73 pgs: 2 peering, 7 active+undersized, 1 active+undersized+degraded, 63 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:07.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:07 smithi042 ceph-mon[108570]: pgmap v398: 73 pgs: 2 peering, 5 active+undersized, 1 active+undersized+degraded, 65 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:07 smithi042 ceph-mon[113926]: pgmap v398: 73 pgs: 2 peering, 5 active+undersized, 1 active+undersized+degraded, 65 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:07 smithi159 ceph-mon[102519]: pgmap v398: 73 pgs: 2 peering, 5 active+undersized, 1 active+undersized+degraded, 65 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%) 2024-09-13T14:40:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:08 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-09-13T14:40:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:08 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:40:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:08 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-09-13T14:40:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:08 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:40:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:08 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 1 pg degraded) 2024-09-13T14:40:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:08 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:40:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:09 smithi042 ceph-mon[113926]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:09 smithi042 ceph-mon[108570]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:09] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-13T14:40:10.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:09 smithi159 ceph-mon[102519]: pgmap v399: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:11 smithi159 ceph-mon[102519]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:11 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:11 smithi042 ceph-mon[113926]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:11 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:11 smithi042 ceph-mon[108570]: pgmap v400: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:12.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:11 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:14.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:13 smithi042 ceph-mon[108570]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:14.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:13 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:40:14.186 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:13 smithi042 ceph-mon[113926]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:14.186 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:13 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:40:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:13 smithi159 ceph-mon[102519]: pgmap v401: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:13 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T14:40:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:14 smithi159 ceph-mon[102519]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:40:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:14 smithi159 ceph-mon[102519]: Upgrade: osd.3 is safe to restart 2024-09-13T14:40:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:14 smithi042 ceph-mon[108570]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:40:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:14 smithi042 ceph-mon[108570]: Upgrade: osd.3 is safe to restart 2024-09-13T14:40:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:14 smithi042 ceph-mon[113926]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T14:40:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:14 smithi042 ceph-mon[113926]: Upgrade: osd.3 is safe to restart 2024-09-13T14:40:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[108570]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[108570]: Upgrade: Updating osd.3 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[113926]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[113926]: Upgrade: Updating osd.3 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:40:16.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:16.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:16 smithi159 ceph-mon[102519]: pgmap v402: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:16 smithi159 ceph-mon[102519]: Upgrade: Updating osd.3 2024-09-13T14:40:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T14:40:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:17 smithi042 ceph-mon[113926]: Deploying daemon osd.3 on smithi042 2024-09-13T14:40:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:17 smithi042 ceph-mon[108570]: Deploying daemon osd.3 on smithi042 2024-09-13T14:40:17.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:17 smithi159 ceph-mon[102519]: Deploying daemon osd.3 on smithi042 2024-09-13T14:40:17.774 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T14:40:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:18 smithi042 ceph-mon[113926]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:18.348 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:18 smithi042 systemd[1]: Stopping Ceph osd.3 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:40:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:18 smithi042 ceph-mon[108570]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:18.434 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T14:40:18.434 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.a smithi042 *:9093,9094 running (13m) 24s ago 20m 18.3M - 0.27.0 11f11916f8cd 9d75438ff3db 2024-09-13T14:40:18.434 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.a smithi159 *:3000 running (12m) 56s ago 20m 78.5M - c8b91775d855 3a8b5443f86c 2024-09-13T14:40:18.434 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi042.pkkbnw smithi042 running (19m) 24s ago 19m 25.4M - 18.2.4-843-g2c01458a c129f5a2fd4e 9ef261615617 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foo.smithi159.ypxvst smithi159 running (19m) 56s ago 19m 21.2M - 18.2.4-843-g2c01458a c129f5a2fd4e 3ba6d39403a4 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.x smithi159 *:8443,9283,8765 running (14m) 56s ago 24m 458M - 19.3.0-4697-gd58f7306 fec07016a2b0 66bfd809a852 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.y smithi042 *:8443,9283,8765 running (15m) 24s ago 26m 522M - 19.3.0-4697-gd58f7306 fec07016a2b0 06039da613e4 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a smithi042 running (12m) 24s ago 26m 70.2M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 69b5986addfb 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b smithi159 running (9m) 56s ago 24m 47.0M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 d2744427c0ab 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c smithi042 running (11m) 24s ago 24m 50.7M 2048M 19.3.0-4697-gd58f7306 fec07016a2b0 251d2b2bf575 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.a smithi042 *:9100 running (14m) 24s ago 20m 11.2M - 1.7.0 72c9c2088986 9c4447f8ffce 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.b smithi159 *:9100 running (14m) 56s ago 20m 9.92M - 1.7.0 72c9c2088986 4e1a6b244f71 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (5m) 24s ago 24m 600M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 771c389bba87 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (4m) 24s ago 23m 331M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 3806b8398b08 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (28s) 24s ago 23m 12.2M 2481M 19.3.0-4697-gd58f7306 fec07016a2b0 247a1eeed07e 2024-09-13T14:40:18.435 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (23m) 24s ago 23m 1416M 2481M 18.2.4-843-g2c01458a c129f5a2fd4e e4959adb228f 2024-09-13T14:40:18.436 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi159 running (22m) 56s ago 22m 596M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 9e81d0713067 2024-09-13T14:40:18.436 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi159 running (22m) 56s ago 22m 976M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 56133b64527d 2024-09-13T14:40:18.436 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi159 running (21m) 56s ago 21m 670M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 0aceebef35ae 2024-09-13T14:40:18.436 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi159 running (21m) 56s ago 21m 543M 2480M 18.2.4-843-g2c01458a c129f5a2fd4e 386bd79b6866 2024-09-13T14:40:18.436 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.a smithi159 *:9095 running (13m) 56s ago 20m 49.8M - 2.51.0 1d3b7f56885b e6c37503183b 2024-09-13T14:40:18.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:18 smithi159 ceph-mon[102519]: pgmap v403: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:18.791 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:40:18.792 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T14:40:18.792 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:40:18.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:40:18.792 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T14:40:18.792 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 2 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 5, 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 3 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:40:18.793 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 2 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 18.2.4-843-g2c01458a (2c01458affee2b2435cd231b6e392435c1c21661) reef (stable)": 7, 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4697-gd58f7306 (d58f73069a8157a9ace12a0247ee3e8497707ea9) squid (dev)": 8 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:40:18.794 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:40:18.795 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[63377]: 2024-09-13T14:40:18.486+0000 7f5d772ee640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:40:18.795 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[63377]: 2024-09-13T14:40:18.486+0000 7f5d772ee640 -1 osd.3 127 *** Got signal Terminated *** 2024-09-13T14:40:18.795 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[63377]: 2024-09-13T14:40:18.487+0000 7f5d772ee640 -1 osd.3 127 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T14:40:19.100 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "3/8 daemons upgraded", 2024-09-13T14:40:19.101 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T14:40:19.101 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T14:40:19.101 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:40:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: osd.3 marked itself down and dead 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/261015983' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: osd.3 marked itself down and dead 2024-09-13T14:40:19.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/261015983' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: from='client.44480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: from='client.44486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: pgmap v404: 73 pgs: 73 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: osd.3 marked itself down and dead 2024-09-13T14:40:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/261015983' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:20.077 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:19] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-13T14:40:20.078 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:19 smithi042 podman[143992]: 2024-09-13 14:40:19.771593743 +0000 UTC m=+1.414066358 container died e4959adb228f62554194feb268527769b4a2b7d32603073bdbd07d1b61706845 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=reef-2c01458) 2024-09-13T14:40:20.348 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:20 smithi042 podman[143992]: 2024-09-13 14:40:20.186253636 +0000 UTC m=+1.828726254 container cleanup e4959adb228f62554194feb268527769b4a2b7d32603073bdbd07d1b61706845 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T14:40:20.348 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:20 smithi042 bash[143992]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: osdmap e128: 8 total, 7 up, 8 in 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='client.44504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]} : dispatch 2024-09-13T14:40:20.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]} : dispatch 2024-09-13T14:40:20.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:20 smithi042 podman[144062]: 2024-09-13 14:40:20.438137766 +0000 UTC m=+0.668373990 container remove e4959adb228f62554194feb268527769b4a2b7d32603073bdbd07d1b61706845 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:40:21.159 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:20 smithi042 podman[144157]: 2024-09-13 14:40:20.760424703 +0000 UTC m=+0.017302864 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:40:21.159 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:20 smithi042 podman[144157]: 2024-09-13 14:40:20.905384131 +0000 UTC m=+0.162262273 container create 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:40:21.160 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:21 smithi042 podman[144157]: 2024-09-13 14:40:21.159109222 +0000 UTC m=+0.415987365 container init 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2024-09-13T14:40:21.160 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: pgmap v407: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]}]': finished 2024-09-13T14:40:21.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]}]': finished 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: pgmap v407: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:21.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.3", "id": [4, 3]}]': finished 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.13", "id": [4, 1]}]': finished 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 2]}]': finished 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 3]}]': finished 2024-09-13T14:40:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 3]}]': finished 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 3]}]': finished 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 3]}]': finished 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 3]}]': finished 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5, 4, 3]}]': finished 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: pgmap v407: 73 pgs: 13 stale+active+clean, 60 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:21.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:21 smithi042 podman[144157]: 2024-09-13 14:40:21.164698881 +0000 UTC m=+0.421577026 container start 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:40:21.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:21 smithi042 podman[144157]: 2024-09-13 14:40:21.258942559 +0000 UTC m=+0.515820712 container attach 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-13T14:40:21.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:21 smithi042 podman[144157]: 2024-09-13 14:40:21.361198373 +0000 UTC m=+0.618076516 container died 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T14:40:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-mon[108570]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T14:40:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-mon[113926]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T14:40:22.348 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144200]: 2024-09-13 14:40:22.046152936 +0000 UTC m=+0.679268431 container remove 44925a560bca3b979b0e4ae8781b7837d7717b8e189b7c7d8a3e2188962ad1f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T14:40:22.349 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T14:40:22.349 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 systemd[1]: Stopped Ceph osd.3 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:40:22.349 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.3.service: Consumed 29min 59.765s CPU time. 2024-09-13T14:40:22.349 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 systemd[1]: Starting Ceph osd.3 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:40:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:22 smithi159 ceph-mon[102519]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T14:40:22.775 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144306]: 2024-09-13 14:40:22.408286235 +0000 UTC m=+0.016507264 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:40:22.776 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144306]: 2024-09-13 14:40:22.513794379 +0000 UTC m=+0.122015398 container create 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS) 2024-09-13T14:40:22.776 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144306]: 2024-09-13 14:40:22.707778112 +0000 UTC m=+0.315999144 container init 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T14:40:22.776 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144306]: 2024-09-13 14:40:22.712504512 +0000 UTC m=+0.320725532 container start 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T14:40:22.776 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 podman[144306]: 2024-09-13 14:40:22.774279976 +0000 UTC m=+0.382501008 container attach 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T14:40:23.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:40:23.102 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:22 smithi042 bash[144306]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-13T14:40:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:23 smithi159 ceph-mon[102519]: pgmap v410: 73 pgs: 4 active+clean+remapped, 1 remapped+peering, 15 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 46 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%); 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T14:40:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:23 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:40:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:23 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:40:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[108570]: pgmap v410: 73 pgs: 4 active+clean+remapped, 1 remapped+peering, 15 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 46 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%); 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T14:40:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:40:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:40:23.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[113926]: pgmap v410: 73 pgs: 4 active+clean+remapped, 1 remapped+peering, 15 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 46 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1/78 objects degraded (1.282%); 8 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T14:40:23.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:40:23.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 1/78 objects degraded (1.282%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:40:23.599 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:40:23.599 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 bash[144306]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 bash[144306]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 bash[144306]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 bash[144306]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T14:40:23.600 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate[144320]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:40:23.601 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 bash[144306]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-13T14:40:23.601 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 podman[144306]: 2024-09-13 14:40:23.285483045 +0000 UTC m=+0.893704065 container died 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T14:40:24.264 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:23 smithi042 podman[144447]: 2024-09-13 14:40:23.911749045 +0000 UTC m=+0.620140107 container remove 369f1c47442bb09c6dcfb2dfb5b6bb56788f48a850e91ba7d8fc18ded784ffee (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T14:40:24.264 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 podman[144516]: 2024-09-13 14:40:24.027627291 +0000 UTC m=+0.016698100 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:40:24.264 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 podman[144516]: 2024-09-13 14:40:24.129224321 +0000 UTC m=+0.118295130 container create 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T14:40:24.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 podman[144516]: 2024-09-13 14:40:24.365496973 +0000 UTC m=+0.354567783 container init 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731) 2024-09-13T14:40:24.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 podman[144516]: 2024-09-13 14:40:24.370491781 +0000 UTC m=+0.359562591 container start 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T14:40:24.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 ceph-osd[144554]: -- 172.21.15.42:0/4169801536 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e54aac3a40 con 0x55e54aaa1c00 2024-09-13T14:40:24.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 bash[144516]: 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 2024-09-13T14:40:24.849 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:24 smithi042 systemd[1]: Started Ceph osd.3 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:40:25.769 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[113926]: pgmap v411: 73 pgs: 8 active+clean+remapped, 1 remapped+peering, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:25.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:25.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[108570]: pgmap v411: 73 pgs: 8 active+clean+remapped, 1 remapped+peering, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:25.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:25.845 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:25 smithi159 ceph-mon[102519]: pgmap v411: 73 pgs: 8 active+clean+remapped, 1 remapped+peering, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:25.845 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.845 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.846 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:25.846 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:26.098 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T14:40:25.767+0000 7f5dbd7fb640 -1 Falling back to public interface 2024-09-13T14:40:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:26 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:40:26.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:26 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:40:26.900 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:26 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:40:27.847 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:27 smithi042 ceph-mon[108570]: pgmap v412: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 20 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:27.847 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:27 smithi042 ceph-mon[113926]: pgmap v412: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 20 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:27.873 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:27 smithi159 ceph-mon[102519]: pgmap v412: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 20 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: pgmap v413: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 15 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.372 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: pgmap v413: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 15 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.373 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: pgmap v413: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 15 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:29.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:29] "GET /metrics HTTP/1.1" 200 33984 "" "Prometheus/2.51.0" 2024-09-13T14:40:31.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:31 smithi159 ceph-mon[102519]: pgmap v414: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 12 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:31.523 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:31 smithi042 ceph-mon[113926]: pgmap v414: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 12 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:31.523 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:31.524 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:31 smithi042 ceph-mon[108570]: pgmap v414: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 12 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:40:31.524 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:40:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: pgmap v415: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 11 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: Upgrade: Finalizing container_image settings 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:40:33.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:40:33.502 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: Upgrade: Complete! 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:40:33.503 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.504 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: pgmap v415: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 11 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: Upgrade: Finalizing container_image settings 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:40:33.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: Upgrade: Complete! 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.602 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: pgmap v415: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 11 B/s, 0 keys/s, 0 objects/s recovering 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: Upgrade: Setting container_image for all nvmeof 2024-09-13T14:40:33.603 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: Upgrade: Finalizing container_image settings 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T14:40:33.604 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.605 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: Upgrade: Complete! 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.606 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:40:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:40:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:40:33.607 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:34.848 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:34 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T14:40:34.449+0000 7f5dbd7fb640 -1 osd.3 127 log_to_monitors true 2024-09-13T14:40:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:35 smithi159 ceph-mon[102519]: pgmap v416: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 7 B/s, 0 objects/s recovering 2024-09-13T14:40:35.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:35 smithi159 ceph-mon[102519]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:40:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:35 smithi042 ceph-mon[108570]: pgmap v416: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 7 B/s, 0 objects/s recovering 2024-09-13T14:40:35.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:35 smithi042 ceph-mon[108570]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:40:35.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:35 smithi042 ceph-mon[113926]: pgmap v416: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%); 7 B/s, 0 objects/s recovering 2024-09-13T14:40:35.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:35 smithi042 ceph-mon[113926]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T14:40:35.599 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 14:40:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T14:40:35.178+0000 7f5db55a5640 -1 osd.3 127 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: osdmap e132: 8 total, 7 up, 8 in 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:40:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: osdmap e132: 8 total, 7 up, 8 in 2024-09-13T14:40:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: osdmap e132: 8 total, 7 up, 8 in 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: from='osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: pgmap v418: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048] boot 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:40:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:37 smithi159 ceph-mon[102519]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: pgmap v418: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048] boot 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:40:37.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[113926]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: pgmap v418: 73 pgs: 9 active+clean+remapped, 28 active+undersized, 2 active+undersized+degraded, 34 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: osd.3 [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048] boot 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T14:40:37.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:37 smithi042 ceph-mon[108570]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T14:40:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:39 smithi159 ceph-mon[102519]: pgmap v421: 73 pgs: 4 remapped+peering, 7 peering, 5 active+clean+remapped, 18 active+undersized, 2 active+undersized+degraded, 37 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:39 smithi159 ceph-mon[102519]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:40:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:39 smithi042 ceph-mon[113926]: pgmap v421: 73 pgs: 4 remapped+peering, 7 peering, 5 active+clean+remapped, 18 active+undersized, 2 active+undersized+degraded, 37 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:39 smithi042 ceph-mon[113926]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:40:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:39 smithi042 ceph-mon[108570]: pgmap v421: 73 pgs: 4 remapped+peering, 7 peering, 5 active+clean+remapped, 18 active+undersized, 2 active+undersized+degraded, 37 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:39.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:39 smithi042 ceph-mon[108570]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T14:40:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:39] "GET /metrics HTTP/1.1" 200 33835 "" "Prometheus/2.51.0" 2024-09-13T14:40:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:41 smithi159 ceph-mon[102519]: pgmap v423: 73 pgs: 4 remapped+peering, 7 peering, 3 active+clean+remapped, 13 active+undersized, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:41 smithi042 ceph-mon[113926]: pgmap v423: 73 pgs: 4 remapped+peering, 7 peering, 3 active+clean+remapped, 13 active+undersized, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:41 smithi042 ceph-mon[108570]: pgmap v423: 73 pgs: 4 remapped+peering, 7 peering, 3 active+clean+remapped, 13 active+undersized, 2 active+undersized+degraded, 44 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 2/78 objects misplaced (2.564%) 2024-09-13T14:40:42.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:42 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded) 2024-09-13T14:40:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:42 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:40:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:42 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded) 2024-09-13T14:40:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:42 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:40:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:42 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 2 pgs degraded) 2024-09-13T14:40:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:42 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:40:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:43 smithi159 ceph-mon[102519]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:43 smithi042 ceph-mon[108570]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:43 smithi042 ceph-mon[113926]: pgmap v424: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:45 smithi159 ceph-mon[102519]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:45 smithi042 ceph-mon[108570]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:45.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:45 smithi042 ceph-mon[113926]: pgmap v425: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:47.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:47 smithi159 ceph-mon[102519]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:47 smithi042 ceph-mon[108570]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:47.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:47 smithi042 ceph-mon[113926]: pgmap v426: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:49.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:49 smithi159 ceph-mon[102519]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:49 smithi042 ceph-mon[108570]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:49 smithi042 ceph-mon[113926]: pgmap v427: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:49] "GET /metrics HTTP/1.1" 200 33835 "" "Prometheus/2.51.0" 2024-09-13T14:40:50.221 DEBUG:teuthology.parallel:result is None 2024-09-13T14:40:50.221 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:40:50.231 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-13T14:40:50.231 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T14:40:50.242 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T14:40:50.242 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T14:40:50.242 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T14:40:50.461 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:50.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:50 smithi159 ceph-mon[102519]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:50.582 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:50 smithi042 ceph-mon[108570]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:50.583 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:50 smithi042 ceph-mon[113926]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[108570]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[113926]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:51.931 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:51 smithi159 ceph-mon[102519]: pgmap v428: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:40:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:40:52.379 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:40:53.036 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3052749594' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:40:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3052749594' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:40:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3052749594' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-13T14:40:53.141 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":42,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6800","nonce":376699412},{"type":"v1","addr":"172.21.15.42:6801","nonce":376699412}]},"active_addr":"172.21.15.42:6801/376699412","active_change":"2024-09-13T14:28:19.830098+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":34101,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":85,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":1771022679}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":3751671095}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":2452907114}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":887557584}]}]} 2024-09-13T14:40:53.146 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T14:40:53.146 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T14:40:53.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:40:53.356 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:53.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:53 smithi159 ceph-mon[102519]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:53 smithi042 ceph-mon[108570]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:53 smithi042 ceph-mon[113926]: pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:55.313 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:40:55.313 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":135,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:40:38.161873+0000","last_up_change":"2024-09-13T14:40:36.157915+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":134,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6803","nonce":3277294232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6805","nonce":3277294232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6809","nonce":3277294232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6807","nonce":3277294232}]},"public_addr":"172.21.15.42:6803/3277294232","cluster_addr":"172.21.15.42:6805/3277294232","heartbeat_back_addr":"172.21.15.42:6809/3277294232","heartbeat_front_addr":"172.21.15.42:6807/3277294232","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":105,"up_from":109,"up_thru":133,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6811","nonce":192065102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6813","nonce":192065102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6817","nonce":192065102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6815","nonce":192065102}]},"public_addr":"172.21.15.42:6811/192065102","cluster_addr":"172.21.15.42:6813/192065102","heartbeat_back_addr":"172.21.15.42:6817/192065102","heartbeat_front_addr":"172.21.15.42:6815/192065102","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":134,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6819","nonce":3631420758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6821","nonce":3631420758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6825","nonce":3631420758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6823","nonce":3631420758}]},"public_addr":"172.21.15.42:6819/3631420758","cluster_addr":"172.21.15.42:6821/3631420758","heartbeat_back_addr":"172.21.15.42:6825/3631420758","heartbeat_front_addr":"172.21.15.42:6823/3631420758","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":127,"up_from":133,"up_thru":134,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6827","nonce":1641847048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6829","nonce":1641847048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6833","nonce":1641847048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6831","nonce":1641847048}]},"public_addr":"172.21.15.42:6827/1641847048","cluster_addr":"172.21.15.42:6829/1641847048","heartbeat_back_addr":"172.21.15.42:6833/1641847048","heartbeat_front_addr":"172.21.15.42:6831/1641847048","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":133,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":133,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":134,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":129,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":105},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.3","mappings":[{"from":4,"to":3}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.13","mappings":[{"from":4,"to":1}]},{"pgid":"3.15","mappings":[{"from":7,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":4,"to":3}]},{"pgid":"4.2","mappings":[{"from":1,"to":3}]},{"pgid":"4.b","mappings":[{"from":1,"to":3}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":3}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5},{"from":4,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.159:0/2982853112":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/1962929896":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.159:6835/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6800/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6800/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.159:0/3438146012":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/1529484914":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/1729697261":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6801/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/471235645":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.159:6834/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/1816541453":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/2764224015":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/800406081":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.159:0/2701547919":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/3559778733":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/2154994614":"2024-09-14T14:28:19.829303+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/1459960260":"2024-09-14T14:28:19.829303+0000","172.21.15.42:6801/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.159:0/2991823830":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:40:55.916 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:55 smithi042 ceph-mon[108570]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:55.916 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:55 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3387969894' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:55.917 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:55 smithi042 ceph-mon[113926]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:55.917 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:55 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3387969894' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:55.939 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T14:40:55.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T14:40:56.142 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:55 smithi159 ceph-mon[102519]: pgmap v430: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:55 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3387969894' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:57 smithi042 ceph-mon[108570]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:57 smithi042 ceph-mon[113926]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:58.135 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:40:58.135 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":135,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T14:40:38.161873+0000","last_up_change":"2024-09-13T14:40:36.157915+0000","last_in_change":"2024-09-13T14:18:38.509228+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":134,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6803","nonce":3277294232}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6805","nonce":3277294232}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6809","nonce":3277294232}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":3277294232},{"type":"v1","addr":"172.21.15.42:6807","nonce":3277294232}]},"public_addr":"172.21.15.42:6803/3277294232","cluster_addr":"172.21.15.42:6805/3277294232","heartbeat_back_addr":"172.21.15.42:6809/3277294232","heartbeat_front_addr":"172.21.15.42:6807/3277294232","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":105,"up_from":109,"up_thru":133,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6811","nonce":192065102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6813","nonce":192065102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6817","nonce":192065102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6815","nonce":192065102}]},"public_addr":"172.21.15.42:6811/192065102","cluster_addr":"172.21.15.42:6813/192065102","heartbeat_back_addr":"172.21.15.42:6817/192065102","heartbeat_front_addr":"172.21.15.42:6815/192065102","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":134,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6819","nonce":3631420758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6821","nonce":3631420758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6825","nonce":3631420758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6823","nonce":3631420758}]},"public_addr":"172.21.15.42:6819/3631420758","cluster_addr":"172.21.15.42:6821/3631420758","heartbeat_back_addr":"172.21.15.42:6825/3631420758","heartbeat_front_addr":"172.21.15.42:6823/3631420758","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":127,"up_from":133,"up_thru":134,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6827","nonce":1641847048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6829","nonce":1641847048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6833","nonce":1641847048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6831","nonce":1641847048}]},"public_addr":"172.21.15.42:6827/1641847048","cluster_addr":"172.21.15.42:6829/1641847048","heartbeat_back_addr":"172.21.15.42:6833/1641847048","heartbeat_front_addr":"172.21.15.42:6831/1641847048","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":133,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":133,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":134,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":129,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":105},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.3","mappings":[{"from":4,"to":3}]},{"pgid":"3.a","mappings":[{"from":4,"to":3}]},{"pgid":"3.13","mappings":[{"from":4,"to":1}]},{"pgid":"3.15","mappings":[{"from":7,"to":2}]},{"pgid":"3.1a","mappings":[{"from":4,"to":3}]},{"pgid":"3.1b","mappings":[{"from":4,"to":3}]},{"pgid":"4.2","mappings":[{"from":1,"to":3}]},{"pgid":"4.b","mappings":[{"from":1,"to":3}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":3}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5},{"from":4,"to":3}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/2154994614":"2024-09-14T14:28:19.829303+0000","172.21.15.42:6800/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2764224015":"2024-09-14T14:28:19.829303+0000","172.21.15.159:0/2982853112":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6835/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6800/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1459960260":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/1729697261":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/471235645":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1962929896":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.159:0/1529484914":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/3438146012":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6834/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/1816541453":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.159:0/2701547919":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/3559778733":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/800406081":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.159:0/2991823830":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T14:40:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:57 smithi159 ceph-mon[102519]: pgmap v431: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:40:58.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T14:40:58.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T14:40:58.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T14:40:58.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T14:40:58.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T14:40:58.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T14:40:58.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T14:40:58.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T14:40:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4052775472' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4052775472' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:59.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4052775472' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T14:40:59.379 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.383 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.384 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.385 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.385 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.390 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.390 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.392 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:40:59.778 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:40:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:40:59] "GET /metrics HTTP/1.1" 200 33981 "" "Prometheus/2.51.0" 2024-09-13T14:41:00.030 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:40:59 smithi042 ceph-mon[108570]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:00.031 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:40:59 smithi042 ceph-mon[113926]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:40:59 smithi159 ceph-mon[102519]: pgmap v432: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:01 smithi042 ceph-mon[108570]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:01 smithi042 ceph-mon[113926]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:01 smithi159 ceph-mon[102519]: pgmap v433: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:03 smithi042 ceph-mon[108570]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:03 smithi042 ceph-mon[113926]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:03 smithi159 ceph-mon[102519]: pgmap v434: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:05.071 INFO:teuthology.orchestra.run.smithi042.stdout:128849019163 2024-09-13T14:41:05.072 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T14:41:05.929 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:05 smithi042 ceph-mon[113926]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:05.929 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:05.929 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:05 smithi042 ceph-mon[108570]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:05.929 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:05 smithi159 ceph-mon[102519]: pgmap v435: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:07.534 INFO:teuthology.orchestra.run.smithi042.stdout:403726925901 2024-09-13T14:41:07.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T14:41:08.055 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:07 smithi042 ceph-mon[108570]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:08.055 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:07 smithi042 ceph-mon[113926]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:07 smithi159 ceph-mon[102519]: pgmap v436: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:10.034 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:09 smithi042 ceph-mon[108570]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:10.034 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:09 smithi042 ceph-mon[113926]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:10.035 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:09] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-13T14:41:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:09 smithi159 ceph-mon[102519]: pgmap v437: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:12.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:11 smithi159 ceph-mon[102519]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:11 smithi042 ceph-mon[108570]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:11 smithi042 ceph-mon[113926]: pgmap v438: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:12.392 INFO:teuthology.orchestra.run.smithi042.stdout:468151435328 2024-09-13T14:41:12.393 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T14:41:13.468 INFO:teuthology.orchestra.run.smithi042.stdout:197568495891 2024-09-13T14:41:13.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T14:41:13.773 INFO:teuthology.orchestra.run.smithi042.stdout:571230650376 2024-09-13T14:41:13.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T14:41:14.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:13 smithi042 ceph-mon[113926]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:14.004 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:13 smithi042 ceph-mon[108570]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:13 smithi159 ceph-mon[102519]: pgmap v439: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:14.878 INFO:teuthology.orchestra.run.smithi042.stdout:231928234253 2024-09-13T14:41:14.878 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T14:41:15.300 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:15.940 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:15 smithi042 ceph-mon[113926]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:15.940 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:15 smithi042 ceph-mon[108570]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:15.974 INFO:teuthology.orchestra.run.smithi042.stdout:541165879312 2024-09-13T14:41:15.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T14:41:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:15 smithi159 ceph-mon[102519]: pgmap v440: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:16.434 INFO:teuthology.orchestra.run.smithi042.stdout:163208757529 2024-09-13T14:41:16.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T14:41:16.457 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:17.131 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:17.132 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:17.132 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:17.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:17 smithi042 ceph-mon[113926]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:17.822 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:17 smithi042 ceph-mon[108570]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:17 smithi159 ceph-mon[102519]: pgmap v441: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:19.905 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:19 smithi042 ceph-mon[113926]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:19.905 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:19 smithi042 ceph-mon[108570]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:19.906 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:19] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-13T14:41:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:19 smithi159 ceph-mon[102519]: pgmap v442: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]} : dispatch 2024-09-13T14:41:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"} : dispatch 2024-09-13T14:41:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"} : dispatch 2024-09-13T14:41:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"} : dispatch 2024-09-13T14:41:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]} : dispatch 2024-09-13T14:41:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"} : dispatch 2024-09-13T14:41:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"} : dispatch 2024-09-13T14:41:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"} : dispatch 2024-09-13T14:41:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:41:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2024-09-13T14:41:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[108570]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"}]': finished 2024-09-13T14:41:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:21 smithi042 ceph-mon[113926]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:41:22.116 INFO:teuthology.orchestra.run.smithi042.stdout:231928234254 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: pgmap v443: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.2"}]': finished 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.b"}]': finished 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "4.12"}]': finished 2024-09-13T14:41:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:21 smithi159 ceph-mon[102519]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T14:41:22.254 INFO:teuthology.orchestra.run.smithi042.stdout:468151435330 2024-09-13T14:41:22.889 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:22.889 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:23.026 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[113926]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:41:23.026 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3901743587' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:41:23.026 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1657432936' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:41:23.026 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[108570]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:41:23.026 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3901743587' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:41:23.027 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1657432936' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:41:23.070 INFO:teuthology.orchestra.run.smithi042.stdout:128849019167 2024-09-13T14:41:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:22 smithi159 ceph-mon[102519]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T14:41:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3901743587' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T14:41:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1657432936' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T14:41:23.825 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234253 got 231928234254 for osd.7 2024-09-13T14:41:23.826 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:23 smithi042 ceph-mon[108570]: pgmap v446: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:23 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1965679281' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:41:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:23 smithi042 ceph-mon[113926]: pgmap v446: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:23 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1965679281' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:41:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:23 smithi159 ceph-mon[102519]: pgmap v446: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:23 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1965679281' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T14:41:24.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019163 got 128849019167 for osd.4 2024-09-13T14:41:24.391 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:24.585 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:24.764 INFO:teuthology.orchestra.run.smithi042.stdout:571230650379 2024-09-13T14:41:25.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:24 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4136506272' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:41:25.075 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:24 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4136506272' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:41:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:24 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4136506272' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T14:41:25.755 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435328 got 468151435330 for osd.1 2024-09-13T14:41:25.755 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:25.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:25 smithi042 ceph-mon[113926]: pgmap v447: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:26.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:25 smithi042 ceph-mon[108570]: pgmap v447: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:25 smithi159 ceph-mon[102519]: pgmap v447: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:26.665 INFO:tasks.cephadm.ceph_manager.ceph:need seq 571230650376 got 571230650379 for osd.3 2024-09-13T14:41:26.666 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:28.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:27 smithi042 ceph-mon[108570]: pgmap v448: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:28.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:27 smithi042 ceph-mon[113926]: pgmap v448: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:27 smithi159 ceph-mon[102519]: pgmap v448: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:29.986 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:29 smithi042 ceph-mon[113926]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:29.986 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:29 smithi042 ceph-mon[108570]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:29.986 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:29] "GET /metrics HTTP/1.1" 200 33982 "" "Prometheus/2.51.0" 2024-09-13T14:41:30.212 INFO:teuthology.orchestra.run.smithi042.stdout:541165879315 2024-09-13T14:41:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:29 smithi159 ceph-mon[102519]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:31.091 INFO:teuthology.orchestra.run.smithi042.stdout:403726925907 2024-09-13T14:41:31.091 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:30 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4201146405' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:41:31.091 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:30 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4201146405' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:41:31.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:30 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4201146405' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T14:41:31.951 INFO:tasks.cephadm.ceph_manager.ceph:need seq 541165879312 got 541165879315 for osd.2 2024-09-13T14:41:31.951 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:32.131 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:31 smithi042 ceph-mon[113926]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:32.132 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1840130161' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:41:32.132 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:31 smithi042 ceph-mon[108570]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:32.132 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1840130161' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:41:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:31 smithi159 ceph-mon[102519]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1840130161' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T14:41:32.314 INFO:teuthology.orchestra.run.smithi042.stdout:197568495894 2024-09-13T14:41:32.642 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925901 got 403726925907 for osd.0 2024-09-13T14:41:32.642 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:32.912 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/985060131' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:41:32.912 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:32 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:32.912 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/985060131' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:41:32.912 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:32 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:33.207 INFO:teuthology.orchestra.run.smithi042.stdout:163208757533 2024-09-13T14:41:33.207 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/985060131' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T14:41:33.207 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:32 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:41:33.495 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495891 got 197568495894 for osd.6 2024-09-13T14:41:33.495 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:34.177 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:33 smithi159 ceph-mon[102519]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:34.177 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2471411526' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:41:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:33 smithi042 ceph-mon[108570]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2471411526' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:41:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:33 smithi042 ceph-mon[113926]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2471411526' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T14:41:34.432 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757529 got 163208757533 for osd.5 2024-09-13T14:41:34.433 DEBUG:teuthology.parallel:result is None 2024-09-13T14:41:34.435 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:41:34.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:41:34.904 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:35.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:35 smithi159 ceph-mon[102519]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:35 smithi042 ceph-mon[113926]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:36.033 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.033 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:35 smithi042 ceph-mon[108570]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:36.033 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:36.758 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:41:36.759 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:41:37.478 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":453,"stamp":"2024-09-13T14:41:36.160577+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1771228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2627260,"kb_used_data":14092,"kb_used_omap":28,"kb_used_meta":2613027,"kb_avail":747333956,"statfs":{"total":767960285184,"available":765269970944,"internally_reserved":0,"allocated":14430208,"data_stored":8775290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":2675740049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002550"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965852+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:41:21.965852+0000","last_peered":"2024-09-13T14:41:21.965852+0000","last_clean":"2024-09-13T14:41:21.965852+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:41:21.965852+0000","last_undegraded":"2024-09-13T14:41:21.965852+0000","last_fullsized":"2024-09-13T14:41:21.965852+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166593+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:41:21.166593+0000","last_peered":"2024-09-13T14:41:21.166593+0000","last_clean":"2024-09-13T14:41:21.166593+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:41:21.166593+0000","last_undegraded":"2024-09-13T14:41:21.166593+0000","last_fullsized":"2024-09-13T14:41:21.166593+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815609+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:41:20.815609+0000","last_peered":"2024-09-13T14:41:20.815609+0000","last_clean":"2024-09-13T14:41:20.815609+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:41:20.815609+0000","last_undegraded":"2024-09-13T14:41:20.815609+0000","last_fullsized":"2024-09-13T14:41:20.815609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822088+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:41:21.822088+0000","last_peered":"2024-09-13T14:41:21.822088+0000","last_clean":"2024-09-13T14:41:21.822088+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:41:21.822088+0000","last_undegraded":"2024-09-13T14:41:21.822088+0000","last_fullsized":"2024-09-13T14:41:21.822088+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822997+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:41:21.822997+0000","last_peered":"2024-09-13T14:41:21.822997+0000","last_clean":"2024-09-13T14:41:21.822997+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:41:21.822997+0000","last_undegraded":"2024-09-13T14:41:21.822997+0000","last_fullsized":"2024-09-13T14:41:21.822997+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166377+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:41:21.166377+0000","last_peered":"2024-09-13T14:41:21.166377+0000","last_clean":"2024-09-13T14:41:21.166377+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:41:21.166377+0000","last_undegraded":"2024-09-13T14:41:21.166377+0000","last_fullsized":"2024-09-13T14:41:21.166377+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965981+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:41:21.965981+0000","last_peered":"2024-09-13T14:41:21.965981+0000","last_clean":"2024-09-13T14:41:21.965981+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:41:21.965981+0000","last_undegraded":"2024-09-13T14:41:21.965981+0000","last_fullsized":"2024-09-13T14:41:21.965981+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818483+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:41:20.818483+0000","last_peered":"2024-09-13T14:41:20.818483+0000","last_clean":"2024-09-13T14:41:20.818483+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:41:20.818483+0000","last_undegraded":"2024-09-13T14:41:20.818483+0000","last_fullsized":"2024-09-13T14:41:20.818483+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166849+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:41:21.166849+0000","last_peered":"2024-09-13T14:41:21.166849+0000","last_clean":"2024-09-13T14:41:21.166849+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:41:21.166849+0000","last_undegraded":"2024-09-13T14:41:21.166849+0000","last_fullsized":"2024-09-13T14:41:21.166849+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815380+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:41:20.815380+0000","last_peered":"2024-09-13T14:41:20.815380+0000","last_clean":"2024-09-13T14:41:20.815380+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:41:20.815380+0000","last_undegraded":"2024-09-13T14:41:20.815380+0000","last_fullsized":"2024-09-13T14:41:20.815380+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166835+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:41:21.166835+0000","last_peered":"2024-09-13T14:41:21.166835+0000","last_clean":"2024-09-13T14:41:21.166835+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:41:21.166835+0000","last_undegraded":"2024-09-13T14:41:21.166835+0000","last_fullsized":"2024-09-13T14:41:21.166835+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819430+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:41:20.819430+0000","last_peered":"2024-09-13T14:41:20.819430+0000","last_clean":"2024-09-13T14:41:20.819430+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:41:20.819430+0000","last_undegraded":"2024-09-13T14:41:20.819430+0000","last_fullsized":"2024-09-13T14:41:20.819430+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166691+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:41:21.166691+0000","last_peered":"2024-09-13T14:41:21.166691+0000","last_clean":"2024-09-13T14:41:21.166691+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:41:21.166691+0000","last_undegraded":"2024-09-13T14:41:21.166691+0000","last_fullsized":"2024-09-13T14:41:21.166691+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818958+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:41:20.818958+0000","last_peered":"2024-09-13T14:41:20.818958+0000","last_clean":"2024-09-13T14:41:20.818958+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:41:20.818958+0000","last_undegraded":"2024-09-13T14:41:20.818958+0000","last_fullsized":"2024-09-13T14:41:20.818958+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815553+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:41:20.815553+0000","last_peered":"2024-09-13T14:41:20.815553+0000","last_clean":"2024-09-13T14:41:20.815553+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:41:20.815553+0000","last_undegraded":"2024-09-13T14:41:20.815553+0000","last_fullsized":"2024-09-13T14:41:20.815553+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.827083+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:41:21.827083+0000","last_peered":"2024-09-13T14:41:21.827083+0000","last_clean":"2024-09-13T14:41:21.827083+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:41:21.827083+0000","last_undegraded":"2024-09-13T14:41:21.827083+0000","last_fullsized":"2024-09-13T14:41:21.827083+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965604+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:41:21.965604+0000","last_peered":"2024-09-13T14:41:21.965604+0000","last_clean":"2024-09-13T14:41:21.965604+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:41:21.965604+0000","last_undegraded":"2024-09-13T14:41:21.965604+0000","last_fullsized":"2024-09-13T14:41:21.965604+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819134+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:41:20.819134+0000","last_peered":"2024-09-13T14:41:20.819134+0000","last_clean":"2024-09-13T14:41:20.819134+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:41:20.819134+0000","last_undegraded":"2024-09-13T14:41:20.819134+0000","last_fullsized":"2024-09-13T14:41:20.819134+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819005+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:41:20.819005+0000","last_peered":"2024-09-13T14:41:20.819005+0000","last_clean":"2024-09-13T14:41:20.819005+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:41:20.819005+0000","last_undegraded":"2024-09-13T14:41:20.819005+0000","last_fullsized":"2024-09-13T14:41:20.819005+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819521+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:41:20.819521+0000","last_peered":"2024-09-13T14:41:20.819521+0000","last_clean":"2024-09-13T14:41:20.819521+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:41:20.819521+0000","last_undegraded":"2024-09-13T14:41:20.819521+0000","last_fullsized":"2024-09-13T14:41:20.819521+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819323+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:41:20.819323+0000","last_peered":"2024-09-13T14:41:20.819323+0000","last_clean":"2024-09-13T14:41:20.819323+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:41:20.819323+0000","last_undegraded":"2024-09-13T14:41:20.819323+0000","last_fullsized":"2024-09-13T14:41:20.819323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965464+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:41:21.965464+0000","last_peered":"2024-09-13T14:41:21.965464+0000","last_clean":"2024-09-13T14:41:21.965464+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:41:21.965464+0000","last_undegraded":"2024-09-13T14:41:21.965464+0000","last_fullsized":"2024-09-13T14:41:21.965464+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821276+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:41:21.821276+0000","last_peered":"2024-09-13T14:41:21.821276+0000","last_clean":"2024-09-13T14:41:21.821276+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:41:21.821276+0000","last_undegraded":"2024-09-13T14:41:21.821276+0000","last_fullsized":"2024-09-13T14:41:21.821276+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":434,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166441+0000","last_change":"2024-09-13T14:34:44.816346+0000","last_active":"2024-09-13T14:41:21.166441+0000","last_peered":"2024-09-13T14:41:21.166441+0000","last_clean":"2024-09-13T14:41:21.166441+0000","last_became_active":"2024-09-13T14:34:44.815951+0000","last_became_peered":"2024-09-13T14:34:44.815951+0000","last_unstale":"2024-09-13T14:41:21.166441+0000","last_undegraded":"2024-09-13T14:41:21.166441+0000","last_fullsized":"2024-09-13T14:41:21.166441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:52:31.648689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815366+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:41:20.815366+0000","last_peered":"2024-09-13T14:41:20.815366+0000","last_clean":"2024-09-13T14:41:20.815366+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:41:20.815366+0000","last_undegraded":"2024-09-13T14:41:20.815366+0000","last_fullsized":"2024-09-13T14:41:20.815366+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965675+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:41:21.965675+0000","last_peered":"2024-09-13T14:41:21.965675+0000","last_clean":"2024-09-13T14:41:21.965675+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:41:21.965675+0000","last_undegraded":"2024-09-13T14:41:21.965675+0000","last_fullsized":"2024-09-13T14:41:21.965675+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818463+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:41:20.818463+0000","last_peered":"2024-09-13T14:41:20.818463+0000","last_clean":"2024-09-13T14:41:20.818463+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:41:20.818463+0000","last_undegraded":"2024-09-13T14:41:20.818463+0000","last_fullsized":"2024-09-13T14:41:20.818463+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":130,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166321+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:41:21.166321+0000","last_peered":"2024-09-13T14:41:21.166321+0000","last_clean":"2024-09-13T14:41:21.166321+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:41:21.166321+0000","last_undegraded":"2024-09-13T14:41:21.166321+0000","last_fullsized":"2024-09-13T14:41:21.166321+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822144+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:41:21.822144+0000","last_peered":"2024-09-13T14:41:21.822144+0000","last_clean":"2024-09-13T14:41:21.822144+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:41:21.822144+0000","last_undegraded":"2024-09-13T14:41:21.822144+0000","last_fullsized":"2024-09-13T14:41:21.822144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822800+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:41:21.822800+0000","last_peered":"2024-09-13T14:41:21.822800+0000","last_clean":"2024-09-13T14:41:21.822800+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:41:21.822800+0000","last_undegraded":"2024-09-13T14:41:21.822800+0000","last_fullsized":"2024-09-13T14:41:21.822800+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.818865+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:41:21.818865+0000","last_peered":"2024-09-13T14:41:21.818865+0000","last_clean":"2024-09-13T14:41:21.818865+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:41:21.818865+0000","last_undegraded":"2024-09-13T14:41:21.818865+0000","last_fullsized":"2024-09-13T14:41:21.818865+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820483+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:41:21.820483+0000","last_peered":"2024-09-13T14:41:21.820483+0000","last_clean":"2024-09-13T14:41:21.820483+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:41:21.820483+0000","last_undegraded":"2024-09-13T14:41:21.820483+0000","last_fullsized":"2024-09-13T14:41:21.820483+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820520+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:41:21.820520+0000","last_peered":"2024-09-13T14:41:21.820520+0000","last_clean":"2024-09-13T14:41:21.820520+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:41:21.820520+0000","last_undegraded":"2024-09-13T14:41:21.820520+0000","last_fullsized":"2024-09-13T14:41:21.820520+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815352+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:41:20.815352+0000","last_peered":"2024-09-13T14:41:20.815352+0000","last_clean":"2024-09-13T14:41:20.815352+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:41:20.815352+0000","last_undegraded":"2024-09-13T14:41:20.815352+0000","last_fullsized":"2024-09-13T14:41:20.815352+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965424+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:41:21.965424+0000","last_peered":"2024-09-13T14:41:21.965424+0000","last_clean":"2024-09-13T14:41:21.965424+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:41:21.965424+0000","last_undegraded":"2024-09-13T14:41:21.965424+0000","last_fullsized":"2024-09-13T14:41:21.965424+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820676+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:41:21.820676+0000","last_peered":"2024-09-13T14:41:21.820676+0000","last_clean":"2024-09-13T14:41:21.820676+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:41:21.820676+0000","last_undegraded":"2024-09-13T14:41:21.820676+0000","last_fullsized":"2024-09-13T14:41:21.820676+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815696+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:41:20.815696+0000","last_peered":"2024-09-13T14:41:20.815696+0000","last_clean":"2024-09-13T14:41:20.815696+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:41:20.815696+0000","last_undegraded":"2024-09-13T14:41:20.815696+0000","last_fullsized":"2024-09-13T14:41:20.815696+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820878+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:41:21.820878+0000","last_peered":"2024-09-13T14:41:21.820878+0000","last_clean":"2024-09-13T14:41:21.820878+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:41:21.820878+0000","last_undegraded":"2024-09-13T14:41:21.820878+0000","last_fullsized":"2024-09-13T14:41:21.820878+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967296+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:41:21.967296+0000","last_peered":"2024-09-13T14:41:21.967296+0000","last_clean":"2024-09-13T14:41:21.967296+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:41:21.967296+0000","last_undegraded":"2024-09-13T14:41:21.967296+0000","last_fullsized":"2024-09-13T14:41:21.967296+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820776+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:41:21.820776+0000","last_peered":"2024-09-13T14:41:21.820776+0000","last_clean":"2024-09-13T14:41:21.820776+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:41:21.820776+0000","last_undegraded":"2024-09-13T14:41:21.820776+0000","last_fullsized":"2024-09-13T14:41:21.820776+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166693+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:41:21.166693+0000","last_peered":"2024-09-13T14:41:21.166693+0000","last_clean":"2024-09-13T14:41:21.166693+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:41:21.166693+0000","last_undegraded":"2024-09-13T14:41:21.166693+0000","last_fullsized":"2024-09-13T14:41:21.166693+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822779+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:41:21.822779+0000","last_peered":"2024-09-13T14:41:21.822779+0000","last_clean":"2024-09-13T14:41:21.822779+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:41:21.822779+0000","last_undegraded":"2024-09-13T14:41:21.822779+0000","last_fullsized":"2024-09-13T14:41:21.822779+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966155+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:41:21.966155+0000","last_peered":"2024-09-13T14:41:21.966155+0000","last_clean":"2024-09-13T14:41:21.966155+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:41:21.966155+0000","last_undegraded":"2024-09-13T14:41:21.966155+0000","last_fullsized":"2024-09-13T14:41:21.966155+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821040+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:41:21.821040+0000","last_peered":"2024-09-13T14:41:21.821040+0000","last_clean":"2024-09-13T14:41:21.821040+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:41:21.821040+0000","last_undegraded":"2024-09-13T14:41:21.821040+0000","last_fullsized":"2024-09-13T14:41:21.821040+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965320+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:41:21.965320+0000","last_peered":"2024-09-13T14:41:21.965320+0000","last_clean":"2024-09-13T14:41:21.965320+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:41:21.965320+0000","last_undegraded":"2024-09-13T14:41:21.965320+0000","last_fullsized":"2024-09-13T14:41:21.965320+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823085+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:41:21.823085+0000","last_peered":"2024-09-13T14:41:21.823085+0000","last_clean":"2024-09-13T14:41:21.823085+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:41:21.823085+0000","last_undegraded":"2024-09-13T14:41:21.823085+0000","last_fullsized":"2024-09-13T14:41:21.823085+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818297+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:41:20.818297+0000","last_peered":"2024-09-13T14:41:20.818297+0000","last_clean":"2024-09-13T14:41:20.818297+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:41:20.818297+0000","last_undegraded":"2024-09-13T14:41:20.818297+0000","last_fullsized":"2024-09-13T14:41:20.818297+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823090+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:41:21.823090+0000","last_peered":"2024-09-13T14:41:21.823090+0000","last_clean":"2024-09-13T14:41:21.823090+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:41:21.823090+0000","last_undegraded":"2024-09-13T14:41:21.823090+0000","last_fullsized":"2024-09-13T14:41:21.823090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819385+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:41:20.819385+0000","last_peered":"2024-09-13T14:41:20.819385+0000","last_clean":"2024-09-13T14:41:20.819385+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:41:20.819385+0000","last_undegraded":"2024-09-13T14:41:20.819385+0000","last_fullsized":"2024-09-13T14:41:20.819385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819252+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:41:20.819252+0000","last_peered":"2024-09-13T14:41:20.819252+0000","last_clean":"2024-09-13T14:41:20.819252+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:41:20.819252+0000","last_undegraded":"2024-09-13T14:41:20.819252+0000","last_fullsized":"2024-09-13T14:41:20.819252+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823984+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:41:21.823984+0000","last_peered":"2024-09-13T14:41:21.823984+0000","last_clean":"2024-09-13T14:41:21.823984+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:41:21.823984+0000","last_undegraded":"2024-09-13T14:41:21.823984+0000","last_fullsized":"2024-09-13T14:41:21.823984+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":37,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819034+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:41:21.819034+0000","last_peered":"2024-09-13T14:41:21.819034+0000","last_clean":"2024-09-13T14:41:21.819034+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:41:21.819034+0000","last_undegraded":"2024-09-13T14:41:21.819034+0000","last_fullsized":"2024-09-13T14:41:21.819034+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815522+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:41:20.815522+0000","last_peered":"2024-09-13T14:41:20.815522+0000","last_clean":"2024-09-13T14:41:20.815522+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:41:20.815522+0000","last_undegraded":"2024-09-13T14:41:20.815522+0000","last_fullsized":"2024-09-13T14:41:20.815522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965405+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:41:21.965405+0000","last_peered":"2024-09-13T14:41:21.965405+0000","last_clean":"2024-09-13T14:41:21.965405+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:41:21.965405+0000","last_undegraded":"2024-09-13T14:41:21.965405+0000","last_fullsized":"2024-09-13T14:41:21.965405+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.824143+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:41:21.824143+0000","last_peered":"2024-09-13T14:41:21.824143+0000","last_clean":"2024-09-13T14:41:21.824143+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:41:21.824143+0000","last_undegraded":"2024-09-13T14:41:21.824143+0000","last_fullsized":"2024-09-13T14:41:21.824143+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965209+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:41:21.965209+0000","last_peered":"2024-09-13T14:41:21.965209+0000","last_clean":"2024-09-13T14:41:21.965209+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:41:21.965209+0000","last_undegraded":"2024-09-13T14:41:21.965209+0000","last_fullsized":"2024-09-13T14:41:21.965209+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820975+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:41:21.820975+0000","last_peered":"2024-09-13T14:41:21.820975+0000","last_clean":"2024-09-13T14:41:21.820975+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:41:21.820975+0000","last_undegraded":"2024-09-13T14:41:21.820975+0000","last_fullsized":"2024-09-13T14:41:21.820975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965426+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:41:21.965426+0000","last_peered":"2024-09-13T14:41:21.965426+0000","last_clean":"2024-09-13T14:41:21.965426+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:41:21.965426+0000","last_undegraded":"2024-09-13T14:41:21.965426+0000","last_fullsized":"2024-09-13T14:41:21.965426+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822906+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:41:21.822906+0000","last_peered":"2024-09-13T14:41:21.822906+0000","last_clean":"2024-09-13T14:41:21.822906+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:41:21.822906+0000","last_undegraded":"2024-09-13T14:41:21.822906+0000","last_fullsized":"2024-09-13T14:41:21.822906+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820591+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:41:21.820591+0000","last_peered":"2024-09-13T14:41:21.820591+0000","last_clean":"2024-09-13T14:41:21.820591+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:41:21.820591+0000","last_undegraded":"2024-09-13T14:41:21.820591+0000","last_fullsized":"2024-09-13T14:41:21.820591+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965443+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:41:21.965443+0000","last_peered":"2024-09-13T14:41:21.965443+0000","last_clean":"2024-09-13T14:41:21.965443+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:41:21.965443+0000","last_undegraded":"2024-09-13T14:41:21.965443+0000","last_fullsized":"2024-09-13T14:41:21.965443+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965492+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:41:21.965492+0000","last_peered":"2024-09-13T14:41:21.965492+0000","last_clean":"2024-09-13T14:41:21.965492+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:41:21.965492+0000","last_undegraded":"2024-09-13T14:41:21.965492+0000","last_fullsized":"2024-09-13T14:41:21.965492+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819327+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:41:21.819327+0000","last_peered":"2024-09-13T14:41:21.819327+0000","last_clean":"2024-09-13T14:41:21.819327+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:41:21.819327+0000","last_undegraded":"2024-09-13T14:41:21.819327+0000","last_fullsized":"2024-09-13T14:41:21.819327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822891+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:41:21.822891+0000","last_peered":"2024-09-13T14:41:21.822891+0000","last_clean":"2024-09-13T14:41:21.822891+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:41:21.822891+0000","last_undegraded":"2024-09-13T14:41:21.822891+0000","last_fullsized":"2024-09-13T14:41:21.822891+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815497+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:41:20.815497+0000","last_peered":"2024-09-13T14:41:20.815497+0000","last_clean":"2024-09-13T14:41:20.815497+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:41:20.815497+0000","last_undegraded":"2024-09-13T14:41:20.815497+0000","last_fullsized":"2024-09-13T14:41:20.815497+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818881+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:41:20.818881+0000","last_peered":"2024-09-13T14:41:20.818881+0000","last_clean":"2024-09-13T14:41:20.818881+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:41:20.818881+0000","last_undegraded":"2024-09-13T14:41:20.818881+0000","last_fullsized":"2024-09-13T14:41:20.818881+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815647+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:41:20.815647+0000","last_peered":"2024-09-13T14:41:20.815647+0000","last_clean":"2024-09-13T14:41:20.815647+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:41:20.815647+0000","last_undegraded":"2024-09-13T14:41:20.815647+0000","last_fullsized":"2024-09-13T14:41:20.815647+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965755+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:41:21.965755+0000","last_peered":"2024-09-13T14:41:21.965755+0000","last_clean":"2024-09-13T14:41:21.965755+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:41:21.965755+0000","last_undegraded":"2024-09-13T14:41:21.965755+0000","last_fullsized":"2024-09-13T14:41:21.965755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965867+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:41:21.965867+0000","last_peered":"2024-09-13T14:41:21.965867+0000","last_clean":"2024-09-13T14:41:21.965867+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:41:21.965867+0000","last_undegraded":"2024-09-13T14:41:21.965867+0000","last_fullsized":"2024-09-13T14:41:21.965867+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967314+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:41:21.967314+0000","last_peered":"2024-09-13T14:41:21.967314+0000","last_clean":"2024-09-13T14:41:21.967314+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:41:21.967314+0000","last_undegraded":"2024-09-13T14:41:21.967314+0000","last_fullsized":"2024-09-13T14:41:21.967314+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":126,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822527+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:41:21.822527+0000","last_peered":"2024-09-13T14:41:21.822527+0000","last_clean":"2024-09-13T14:41:21.822527+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:41:21.822527+0000","last_undegraded":"2024-09-13T14:41:21.822527+0000","last_fullsized":"2024-09-13T14:41:21.822527+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234257,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492644,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":490494,"kb_avail":93252508,"statfs":{"total":95995035648,"available":95490568192,"internally_reserved":0,"allocated":2187264,"data_stored":1468787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":502266315},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495895,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428768,"kb_used_data":2132,"kb_used_omap":1,"kb_used_meta":426622,"kb_avail":93316384,"statfs":{"total":95995035648,"available":95555977216,"internally_reserved":0,"allocated":2183168,"data_stored":1468750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":436861383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757533,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588052,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":586494,"kb_avail":93157100,"statfs":{"total":95995035648,"available":95392870400,"internally_reserved":0,"allocated":1576960,"data_stored":873877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":600570310},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019169,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556064,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":554494,"kb_avail":93189088,"statfs":{"total":95995035648,"available":95425626112,"internally_reserved":0,"allocated":1589248,"data_stored":878405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":567802310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650382,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":85563,"kb_avail":93658032,"statfs":{"total":95995035648,"available":95905824768,"internally_reserved":0,"allocated":1572864,"data_stored":873639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":87617071},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879317,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76820,"kb_used_data":1544,"kb_used_omap":6,"kb_used_meta":75257,"kb_avail":93668332,"statfs":{"total":95995035648,"available":95916371968,"internally_reserved":0,"allocated":1581056,"data_stored":874255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":77063737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435333,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":201676,"kb_used_data":1532,"kb_used_omap":5,"kb_used_meta":200122,"kb_avail":93543476,"statfs":{"total":95995035648,"available":95788519424,"internally_reserved":0,"allocated":1568768,"data_stored":873056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":204925281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925908,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196116,"kb_used_data":2120,"kb_used_omap":5,"kb_used_meta":193978,"kb_avail":93549036,"statfs":{"total":95995035648,"available":95794212864,"internally_reserved":0,"allocated":2170880,"data_stored":1464521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":198633642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:41:37.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:41:38.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[113926]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:38.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[113926]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:38.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.245 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[108570]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:38.245 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[108570]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:38.245 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.245 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:37 smithi159 ceph-mon[102519]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:37 smithi159 ceph-mon[102519]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:38.367 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:40.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:39 smithi042 ceph-mon[108570]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:40.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:39 smithi042 ceph-mon[113926]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:40.014 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:39] "GET /metrics HTTP/1.1" 200 33981 "" "Prometheus/2.51.0" 2024-09-13T14:41:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:39 smithi159 ceph-mon[102519]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:41.077 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:41:41.077 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[108570]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:41.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.708 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:41 smithi042 ceph-mon[113926]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:41.728 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":455,"stamp":"2024-09-13T14:41:40.161498+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1771228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2627260,"kb_used_data":14092,"kb_used_omap":28,"kb_used_meta":2613027,"kb_avail":747333956,"statfs":{"total":767960285184,"available":765269970944,"internally_reserved":0,"allocated":14430208,"data_stored":8775290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":2675740049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002594"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965852+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:41:21.965852+0000","last_peered":"2024-09-13T14:41:21.965852+0000","last_clean":"2024-09-13T14:41:21.965852+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:41:21.965852+0000","last_undegraded":"2024-09-13T14:41:21.965852+0000","last_fullsized":"2024-09-13T14:41:21.965852+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166593+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:41:21.166593+0000","last_peered":"2024-09-13T14:41:21.166593+0000","last_clean":"2024-09-13T14:41:21.166593+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:41:21.166593+0000","last_undegraded":"2024-09-13T14:41:21.166593+0000","last_fullsized":"2024-09-13T14:41:21.166593+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815609+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:41:20.815609+0000","last_peered":"2024-09-13T14:41:20.815609+0000","last_clean":"2024-09-13T14:41:20.815609+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:41:20.815609+0000","last_undegraded":"2024-09-13T14:41:20.815609+0000","last_fullsized":"2024-09-13T14:41:20.815609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822088+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:41:21.822088+0000","last_peered":"2024-09-13T14:41:21.822088+0000","last_clean":"2024-09-13T14:41:21.822088+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:41:21.822088+0000","last_undegraded":"2024-09-13T14:41:21.822088+0000","last_fullsized":"2024-09-13T14:41:21.822088+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822997+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:41:21.822997+0000","last_peered":"2024-09-13T14:41:21.822997+0000","last_clean":"2024-09-13T14:41:21.822997+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:41:21.822997+0000","last_undegraded":"2024-09-13T14:41:21.822997+0000","last_fullsized":"2024-09-13T14:41:21.822997+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166377+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:41:21.166377+0000","last_peered":"2024-09-13T14:41:21.166377+0000","last_clean":"2024-09-13T14:41:21.166377+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:41:21.166377+0000","last_undegraded":"2024-09-13T14:41:21.166377+0000","last_fullsized":"2024-09-13T14:41:21.166377+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965981+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:41:21.965981+0000","last_peered":"2024-09-13T14:41:21.965981+0000","last_clean":"2024-09-13T14:41:21.965981+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:41:21.965981+0000","last_undegraded":"2024-09-13T14:41:21.965981+0000","last_fullsized":"2024-09-13T14:41:21.965981+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818483+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:41:20.818483+0000","last_peered":"2024-09-13T14:41:20.818483+0000","last_clean":"2024-09-13T14:41:20.818483+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:41:20.818483+0000","last_undegraded":"2024-09-13T14:41:20.818483+0000","last_fullsized":"2024-09-13T14:41:20.818483+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166849+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:41:21.166849+0000","last_peered":"2024-09-13T14:41:21.166849+0000","last_clean":"2024-09-13T14:41:21.166849+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:41:21.166849+0000","last_undegraded":"2024-09-13T14:41:21.166849+0000","last_fullsized":"2024-09-13T14:41:21.166849+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815380+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:41:20.815380+0000","last_peered":"2024-09-13T14:41:20.815380+0000","last_clean":"2024-09-13T14:41:20.815380+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:41:20.815380+0000","last_undegraded":"2024-09-13T14:41:20.815380+0000","last_fullsized":"2024-09-13T14:41:20.815380+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166835+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:41:21.166835+0000","last_peered":"2024-09-13T14:41:21.166835+0000","last_clean":"2024-09-13T14:41:21.166835+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:41:21.166835+0000","last_undegraded":"2024-09-13T14:41:21.166835+0000","last_fullsized":"2024-09-13T14:41:21.166835+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819430+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:41:20.819430+0000","last_peered":"2024-09-13T14:41:20.819430+0000","last_clean":"2024-09-13T14:41:20.819430+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:41:20.819430+0000","last_undegraded":"2024-09-13T14:41:20.819430+0000","last_fullsized":"2024-09-13T14:41:20.819430+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166691+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:41:21.166691+0000","last_peered":"2024-09-13T14:41:21.166691+0000","last_clean":"2024-09-13T14:41:21.166691+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:41:21.166691+0000","last_undegraded":"2024-09-13T14:41:21.166691+0000","last_fullsized":"2024-09-13T14:41:21.166691+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818958+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:41:20.818958+0000","last_peered":"2024-09-13T14:41:20.818958+0000","last_clean":"2024-09-13T14:41:20.818958+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:41:20.818958+0000","last_undegraded":"2024-09-13T14:41:20.818958+0000","last_fullsized":"2024-09-13T14:41:20.818958+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815553+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:41:20.815553+0000","last_peered":"2024-09-13T14:41:20.815553+0000","last_clean":"2024-09-13T14:41:20.815553+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:41:20.815553+0000","last_undegraded":"2024-09-13T14:41:20.815553+0000","last_fullsized":"2024-09-13T14:41:20.815553+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.827083+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:41:21.827083+0000","last_peered":"2024-09-13T14:41:21.827083+0000","last_clean":"2024-09-13T14:41:21.827083+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:41:21.827083+0000","last_undegraded":"2024-09-13T14:41:21.827083+0000","last_fullsized":"2024-09-13T14:41:21.827083+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965604+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:41:21.965604+0000","last_peered":"2024-09-13T14:41:21.965604+0000","last_clean":"2024-09-13T14:41:21.965604+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:41:21.965604+0000","last_undegraded":"2024-09-13T14:41:21.965604+0000","last_fullsized":"2024-09-13T14:41:21.965604+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819134+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:41:20.819134+0000","last_peered":"2024-09-13T14:41:20.819134+0000","last_clean":"2024-09-13T14:41:20.819134+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:41:20.819134+0000","last_undegraded":"2024-09-13T14:41:20.819134+0000","last_fullsized":"2024-09-13T14:41:20.819134+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819005+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:41:20.819005+0000","last_peered":"2024-09-13T14:41:20.819005+0000","last_clean":"2024-09-13T14:41:20.819005+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:41:20.819005+0000","last_undegraded":"2024-09-13T14:41:20.819005+0000","last_fullsized":"2024-09-13T14:41:20.819005+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819521+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:41:20.819521+0000","last_peered":"2024-09-13T14:41:20.819521+0000","last_clean":"2024-09-13T14:41:20.819521+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:41:20.819521+0000","last_undegraded":"2024-09-13T14:41:20.819521+0000","last_fullsized":"2024-09-13T14:41:20.819521+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819323+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:41:20.819323+0000","last_peered":"2024-09-13T14:41:20.819323+0000","last_clean":"2024-09-13T14:41:20.819323+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:41:20.819323+0000","last_undegraded":"2024-09-13T14:41:20.819323+0000","last_fullsized":"2024-09-13T14:41:20.819323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965464+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:41:21.965464+0000","last_peered":"2024-09-13T14:41:21.965464+0000","last_clean":"2024-09-13T14:41:21.965464+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:41:21.965464+0000","last_undegraded":"2024-09-13T14:41:21.965464+0000","last_fullsized":"2024-09-13T14:41:21.965464+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821276+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:41:21.821276+0000","last_peered":"2024-09-13T14:41:21.821276+0000","last_clean":"2024-09-13T14:41:21.821276+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:41:21.821276+0000","last_undegraded":"2024-09-13T14:41:21.821276+0000","last_fullsized":"2024-09-13T14:41:21.821276+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":434,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166441+0000","last_change":"2024-09-13T14:34:44.816346+0000","last_active":"2024-09-13T14:41:21.166441+0000","last_peered":"2024-09-13T14:41:21.166441+0000","last_clean":"2024-09-13T14:41:21.166441+0000","last_became_active":"2024-09-13T14:34:44.815951+0000","last_became_peered":"2024-09-13T14:34:44.815951+0000","last_unstale":"2024-09-13T14:41:21.166441+0000","last_undegraded":"2024-09-13T14:41:21.166441+0000","last_fullsized":"2024-09-13T14:41:21.166441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:52:31.648689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815366+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:41:20.815366+0000","last_peered":"2024-09-13T14:41:20.815366+0000","last_clean":"2024-09-13T14:41:20.815366+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:41:20.815366+0000","last_undegraded":"2024-09-13T14:41:20.815366+0000","last_fullsized":"2024-09-13T14:41:20.815366+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965675+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:41:21.965675+0000","last_peered":"2024-09-13T14:41:21.965675+0000","last_clean":"2024-09-13T14:41:21.965675+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:41:21.965675+0000","last_undegraded":"2024-09-13T14:41:21.965675+0000","last_fullsized":"2024-09-13T14:41:21.965675+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818463+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:41:20.818463+0000","last_peered":"2024-09-13T14:41:20.818463+0000","last_clean":"2024-09-13T14:41:20.818463+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:41:20.818463+0000","last_undegraded":"2024-09-13T14:41:20.818463+0000","last_fullsized":"2024-09-13T14:41:20.818463+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965746+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:41:21.965746+0000","last_peered":"2024-09-13T14:41:21.965746+0000","last_clean":"2024-09-13T14:41:21.965746+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:41:21.965746+0000","last_undegraded":"2024-09-13T14:41:21.965746+0000","last_fullsized":"2024-09-13T14:41:21.965746+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822144+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:41:21.822144+0000","last_peered":"2024-09-13T14:41:21.822144+0000","last_clean":"2024-09-13T14:41:21.822144+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:41:21.822144+0000","last_undegraded":"2024-09-13T14:41:21.822144+0000","last_fullsized":"2024-09-13T14:41:21.822144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822800+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:41:21.822800+0000","last_peered":"2024-09-13T14:41:21.822800+0000","last_clean":"2024-09-13T14:41:21.822800+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:41:21.822800+0000","last_undegraded":"2024-09-13T14:41:21.822800+0000","last_fullsized":"2024-09-13T14:41:21.822800+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.818865+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:41:21.818865+0000","last_peered":"2024-09-13T14:41:21.818865+0000","last_clean":"2024-09-13T14:41:21.818865+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:41:21.818865+0000","last_undegraded":"2024-09-13T14:41:21.818865+0000","last_fullsized":"2024-09-13T14:41:21.818865+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820483+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:41:21.820483+0000","last_peered":"2024-09-13T14:41:21.820483+0000","last_clean":"2024-09-13T14:41:21.820483+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:41:21.820483+0000","last_undegraded":"2024-09-13T14:41:21.820483+0000","last_fullsized":"2024-09-13T14:41:21.820483+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820520+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:41:21.820520+0000","last_peered":"2024-09-13T14:41:21.820520+0000","last_clean":"2024-09-13T14:41:21.820520+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:41:21.820520+0000","last_undegraded":"2024-09-13T14:41:21.820520+0000","last_fullsized":"2024-09-13T14:41:21.820520+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815352+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:41:20.815352+0000","last_peered":"2024-09-13T14:41:20.815352+0000","last_clean":"2024-09-13T14:41:20.815352+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:41:20.815352+0000","last_undegraded":"2024-09-13T14:41:20.815352+0000","last_fullsized":"2024-09-13T14:41:20.815352+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965424+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:41:21.965424+0000","last_peered":"2024-09-13T14:41:21.965424+0000","last_clean":"2024-09-13T14:41:21.965424+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:41:21.965424+0000","last_undegraded":"2024-09-13T14:41:21.965424+0000","last_fullsized":"2024-09-13T14:41:21.965424+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820676+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:41:21.820676+0000","last_peered":"2024-09-13T14:41:21.820676+0000","last_clean":"2024-09-13T14:41:21.820676+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:41:21.820676+0000","last_undegraded":"2024-09-13T14:41:21.820676+0000","last_fullsized":"2024-09-13T14:41:21.820676+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815696+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:41:20.815696+0000","last_peered":"2024-09-13T14:41:20.815696+0000","last_clean":"2024-09-13T14:41:20.815696+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:41:20.815696+0000","last_undegraded":"2024-09-13T14:41:20.815696+0000","last_fullsized":"2024-09-13T14:41:20.815696+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820878+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:41:21.820878+0000","last_peered":"2024-09-13T14:41:21.820878+0000","last_clean":"2024-09-13T14:41:21.820878+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:41:21.820878+0000","last_undegraded":"2024-09-13T14:41:21.820878+0000","last_fullsized":"2024-09-13T14:41:21.820878+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967296+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:41:21.967296+0000","last_peered":"2024-09-13T14:41:21.967296+0000","last_clean":"2024-09-13T14:41:21.967296+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:41:21.967296+0000","last_undegraded":"2024-09-13T14:41:21.967296+0000","last_fullsized":"2024-09-13T14:41:21.967296+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820776+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:41:21.820776+0000","last_peered":"2024-09-13T14:41:21.820776+0000","last_clean":"2024-09-13T14:41:21.820776+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:41:21.820776+0000","last_undegraded":"2024-09-13T14:41:21.820776+0000","last_fullsized":"2024-09-13T14:41:21.820776+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166693+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:41:21.166693+0000","last_peered":"2024-09-13T14:41:21.166693+0000","last_clean":"2024-09-13T14:41:21.166693+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:41:21.166693+0000","last_undegraded":"2024-09-13T14:41:21.166693+0000","last_fullsized":"2024-09-13T14:41:21.166693+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822779+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:41:21.822779+0000","last_peered":"2024-09-13T14:41:21.822779+0000","last_clean":"2024-09-13T14:41:21.822779+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:41:21.822779+0000","last_undegraded":"2024-09-13T14:41:21.822779+0000","last_fullsized":"2024-09-13T14:41:21.822779+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966155+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:41:21.966155+0000","last_peered":"2024-09-13T14:41:21.966155+0000","last_clean":"2024-09-13T14:41:21.966155+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:41:21.966155+0000","last_undegraded":"2024-09-13T14:41:21.966155+0000","last_fullsized":"2024-09-13T14:41:21.966155+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821040+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:41:21.821040+0000","last_peered":"2024-09-13T14:41:21.821040+0000","last_clean":"2024-09-13T14:41:21.821040+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:41:21.821040+0000","last_undegraded":"2024-09-13T14:41:21.821040+0000","last_fullsized":"2024-09-13T14:41:21.821040+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965320+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:41:21.965320+0000","last_peered":"2024-09-13T14:41:21.965320+0000","last_clean":"2024-09-13T14:41:21.965320+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:41:21.965320+0000","last_undegraded":"2024-09-13T14:41:21.965320+0000","last_fullsized":"2024-09-13T14:41:21.965320+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823085+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:41:21.823085+0000","last_peered":"2024-09-13T14:41:21.823085+0000","last_clean":"2024-09-13T14:41:21.823085+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:41:21.823085+0000","last_undegraded":"2024-09-13T14:41:21.823085+0000","last_fullsized":"2024-09-13T14:41:21.823085+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818297+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:41:20.818297+0000","last_peered":"2024-09-13T14:41:20.818297+0000","last_clean":"2024-09-13T14:41:20.818297+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:41:20.818297+0000","last_undegraded":"2024-09-13T14:41:20.818297+0000","last_fullsized":"2024-09-13T14:41:20.818297+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823090+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:41:21.823090+0000","last_peered":"2024-09-13T14:41:21.823090+0000","last_clean":"2024-09-13T14:41:21.823090+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:41:21.823090+0000","last_undegraded":"2024-09-13T14:41:21.823090+0000","last_fullsized":"2024-09-13T14:41:21.823090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819385+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:41:20.819385+0000","last_peered":"2024-09-13T14:41:20.819385+0000","last_clean":"2024-09-13T14:41:20.819385+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:41:20.819385+0000","last_undegraded":"2024-09-13T14:41:20.819385+0000","last_fullsized":"2024-09-13T14:41:20.819385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819252+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:41:20.819252+0000","last_peered":"2024-09-13T14:41:20.819252+0000","last_clean":"2024-09-13T14:41:20.819252+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:41:20.819252+0000","last_undegraded":"2024-09-13T14:41:20.819252+0000","last_fullsized":"2024-09-13T14:41:20.819252+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823984+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:41:21.823984+0000","last_peered":"2024-09-13T14:41:21.823984+0000","last_clean":"2024-09-13T14:41:21.823984+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:41:21.823984+0000","last_undegraded":"2024-09-13T14:41:21.823984+0000","last_fullsized":"2024-09-13T14:41:21.823984+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":37,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819034+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:41:21.819034+0000","last_peered":"2024-09-13T14:41:21.819034+0000","last_clean":"2024-09-13T14:41:21.819034+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:41:21.819034+0000","last_undegraded":"2024-09-13T14:41:21.819034+0000","last_fullsized":"2024-09-13T14:41:21.819034+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815522+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:41:20.815522+0000","last_peered":"2024-09-13T14:41:20.815522+0000","last_clean":"2024-09-13T14:41:20.815522+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:41:20.815522+0000","last_undegraded":"2024-09-13T14:41:20.815522+0000","last_fullsized":"2024-09-13T14:41:20.815522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965405+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:41:21.965405+0000","last_peered":"2024-09-13T14:41:21.965405+0000","last_clean":"2024-09-13T14:41:21.965405+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:41:21.965405+0000","last_undegraded":"2024-09-13T14:41:21.965405+0000","last_fullsized":"2024-09-13T14:41:21.965405+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.824143+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:41:21.824143+0000","last_peered":"2024-09-13T14:41:21.824143+0000","last_clean":"2024-09-13T14:41:21.824143+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:41:21.824143+0000","last_undegraded":"2024-09-13T14:41:21.824143+0000","last_fullsized":"2024-09-13T14:41:21.824143+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965209+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:41:21.965209+0000","last_peered":"2024-09-13T14:41:21.965209+0000","last_clean":"2024-09-13T14:41:21.965209+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:41:21.965209+0000","last_undegraded":"2024-09-13T14:41:21.965209+0000","last_fullsized":"2024-09-13T14:41:21.965209+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820975+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:41:21.820975+0000","last_peered":"2024-09-13T14:41:21.820975+0000","last_clean":"2024-09-13T14:41:21.820975+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:41:21.820975+0000","last_undegraded":"2024-09-13T14:41:21.820975+0000","last_fullsized":"2024-09-13T14:41:21.820975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965426+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:41:21.965426+0000","last_peered":"2024-09-13T14:41:21.965426+0000","last_clean":"2024-09-13T14:41:21.965426+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:41:21.965426+0000","last_undegraded":"2024-09-13T14:41:21.965426+0000","last_fullsized":"2024-09-13T14:41:21.965426+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822906+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:41:21.822906+0000","last_peered":"2024-09-13T14:41:21.822906+0000","last_clean":"2024-09-13T14:41:21.822906+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:41:21.822906+0000","last_undegraded":"2024-09-13T14:41:21.822906+0000","last_fullsized":"2024-09-13T14:41:21.822906+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820591+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:41:21.820591+0000","last_peered":"2024-09-13T14:41:21.820591+0000","last_clean":"2024-09-13T14:41:21.820591+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:41:21.820591+0000","last_undegraded":"2024-09-13T14:41:21.820591+0000","last_fullsized":"2024-09-13T14:41:21.820591+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965443+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:41:21.965443+0000","last_peered":"2024-09-13T14:41:21.965443+0000","last_clean":"2024-09-13T14:41:21.965443+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:41:21.965443+0000","last_undegraded":"2024-09-13T14:41:21.965443+0000","last_fullsized":"2024-09-13T14:41:21.965443+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965492+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:41:21.965492+0000","last_peered":"2024-09-13T14:41:21.965492+0000","last_clean":"2024-09-13T14:41:21.965492+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:41:21.965492+0000","last_undegraded":"2024-09-13T14:41:21.965492+0000","last_fullsized":"2024-09-13T14:41:21.965492+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819327+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:41:21.819327+0000","last_peered":"2024-09-13T14:41:21.819327+0000","last_clean":"2024-09-13T14:41:21.819327+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:41:21.819327+0000","last_undegraded":"2024-09-13T14:41:21.819327+0000","last_fullsized":"2024-09-13T14:41:21.819327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822891+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:41:21.822891+0000","last_peered":"2024-09-13T14:41:21.822891+0000","last_clean":"2024-09-13T14:41:21.822891+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:41:21.822891+0000","last_undegraded":"2024-09-13T14:41:21.822891+0000","last_fullsized":"2024-09-13T14:41:21.822891+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815497+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:41:20.815497+0000","last_peered":"2024-09-13T14:41:20.815497+0000","last_clean":"2024-09-13T14:41:20.815497+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:41:20.815497+0000","last_undegraded":"2024-09-13T14:41:20.815497+0000","last_fullsized":"2024-09-13T14:41:20.815497+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818881+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:41:20.818881+0000","last_peered":"2024-09-13T14:41:20.818881+0000","last_clean":"2024-09-13T14:41:20.818881+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:41:20.818881+0000","last_undegraded":"2024-09-13T14:41:20.818881+0000","last_fullsized":"2024-09-13T14:41:20.818881+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815647+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:41:20.815647+0000","last_peered":"2024-09-13T14:41:20.815647+0000","last_clean":"2024-09-13T14:41:20.815647+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:41:20.815647+0000","last_undegraded":"2024-09-13T14:41:20.815647+0000","last_fullsized":"2024-09-13T14:41:20.815647+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965755+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:41:21.965755+0000","last_peered":"2024-09-13T14:41:21.965755+0000","last_clean":"2024-09-13T14:41:21.965755+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:41:21.965755+0000","last_undegraded":"2024-09-13T14:41:21.965755+0000","last_fullsized":"2024-09-13T14:41:21.965755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965867+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:41:21.965867+0000","last_peered":"2024-09-13T14:41:21.965867+0000","last_clean":"2024-09-13T14:41:21.965867+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:41:21.965867+0000","last_undegraded":"2024-09-13T14:41:21.965867+0000","last_fullsized":"2024-09-13T14:41:21.965867+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967314+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:41:21.967314+0000","last_peered":"2024-09-13T14:41:21.967314+0000","last_clean":"2024-09-13T14:41:21.967314+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:41:21.967314+0000","last_undegraded":"2024-09-13T14:41:21.967314+0000","last_fullsized":"2024-09-13T14:41:21.967314+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":126,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822527+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:41:21.822527+0000","last_peered":"2024-09-13T14:41:21.822527+0000","last_clean":"2024-09-13T14:41:21.822527+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:41:21.822527+0000","last_undegraded":"2024-09-13T14:41:21.822527+0000","last_fullsized":"2024-09-13T14:41:21.822527+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234258,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492644,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":490494,"kb_avail":93252508,"statfs":{"total":95995035648,"available":95490568192,"internally_reserved":0,"allocated":2187264,"data_stored":1468787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":502266315},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495896,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428768,"kb_used_data":2132,"kb_used_omap":1,"kb_used_meta":426622,"kb_avail":93316384,"statfs":{"total":95995035648,"available":95555977216,"internally_reserved":0,"allocated":2183168,"data_stored":1468750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":436861383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757534,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588052,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":586494,"kb_avail":93157100,"statfs":{"total":95995035648,"available":95392870400,"internally_reserved":0,"allocated":1576960,"data_stored":873877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":600570310},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019170,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556064,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":554494,"kb_avail":93189088,"statfs":{"total":95995035648,"available":95425626112,"internally_reserved":0,"allocated":1589248,"data_stored":878405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":567802310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650382,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":85563,"kb_avail":93658032,"statfs":{"total":95995035648,"available":95905824768,"internally_reserved":0,"allocated":1572864,"data_stored":873639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":87617071},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879317,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76820,"kb_used_data":1544,"kb_used_omap":6,"kb_used_meta":75257,"kb_avail":93668332,"statfs":{"total":95995035648,"available":95916371968,"internally_reserved":0,"allocated":1581056,"data_stored":874255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":77063737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435334,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":201676,"kb_used_data":1532,"kb_used_omap":5,"kb_used_meta":200122,"kb_avail":93543476,"statfs":{"total":95995035648,"available":95788519424,"internally_reserved":0,"allocated":1568768,"data_stored":873056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":204925281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925909,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196116,"kb_used_data":2120,"kb_used_omap":5,"kb_used_meta":193978,"kb_avail":93549036,"statfs":{"total":95995035648,"available":95794212864,"internally_reserved":0,"allocated":2170880,"data_stored":1464521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":198633642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:41:41.731 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:41:41.731 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T14:41:41.731 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T14:41:41.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T14:41:41.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: pgmap v455: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:41.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:41:41.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:41:41.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:41:41.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:41 smithi159 ceph-mon[102519]: from='client.34722 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:41.934 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:43 smithi042 ceph-mon[108570]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:43 smithi042 ceph-mon[113926]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:43.737 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:41:43.737 INFO:teuthology.orchestra.run.smithi042.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T14:41:43.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:43 smithi159 ceph-mon[102519]: pgmap v456: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:44.416 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T14:41:44.416 INFO:teuthology.run_tasks:Running task print... 2024-09-13T14:41:44.426 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-13T14:41:44.426 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T14:41:44.440 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T14:41:44.440 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T14:41:44.440 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-13T14:41:44.443 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:41:44.443 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'wip-aclamk-cbt-combined'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'wip-aclamk-cbt-combined', 'sha1': 'd58f73069a8157a9ace12a0247ee3e8497707ea9'}} 2024-09-13T14:41:44.443 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-13T14:41:44.443 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-13T14:41:44.443 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T14:41:44.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:41:44.656 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:44.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2336810595' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:41:44.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2336810595' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:41:44.778 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2336810595' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-13T14:41:45.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:45 smithi159 ceph-mon[102519]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:45.761 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:45 smithi042 ceph-mon[113926]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:45.762 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:45 smithi042 ceph-mon[108570]: pgmap v457: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:46.517 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:41:46.517 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:41:47.199 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":458,"stamp":"2024-09-13T14:41:46.162816+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1771228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2627260,"kb_used_data":14092,"kb_used_omap":28,"kb_used_meta":2613027,"kb_avail":747333956,"statfs":{"total":767960285184,"available":765269970944,"internally_reserved":0,"allocated":14430208,"data_stored":8775290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":2675740049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002654"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965852+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:41:21.965852+0000","last_peered":"2024-09-13T14:41:21.965852+0000","last_clean":"2024-09-13T14:41:21.965852+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:41:21.965852+0000","last_undegraded":"2024-09-13T14:41:21.965852+0000","last_fullsized":"2024-09-13T14:41:21.965852+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166593+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:41:21.166593+0000","last_peered":"2024-09-13T14:41:21.166593+0000","last_clean":"2024-09-13T14:41:21.166593+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:41:21.166593+0000","last_undegraded":"2024-09-13T14:41:21.166593+0000","last_fullsized":"2024-09-13T14:41:21.166593+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815609+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:41:20.815609+0000","last_peered":"2024-09-13T14:41:20.815609+0000","last_clean":"2024-09-13T14:41:20.815609+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:41:20.815609+0000","last_undegraded":"2024-09-13T14:41:20.815609+0000","last_fullsized":"2024-09-13T14:41:20.815609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822140+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:41:21.822140+0000","last_peered":"2024-09-13T14:41:21.822140+0000","last_clean":"2024-09-13T14:41:21.822140+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:41:21.822140+0000","last_undegraded":"2024-09-13T14:41:21.822140+0000","last_fullsized":"2024-09-13T14:41:21.822140+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822997+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:41:21.822997+0000","last_peered":"2024-09-13T14:41:21.822997+0000","last_clean":"2024-09-13T14:41:21.822997+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:41:21.822997+0000","last_undegraded":"2024-09-13T14:41:21.822997+0000","last_fullsized":"2024-09-13T14:41:21.822997+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166377+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:41:21.166377+0000","last_peered":"2024-09-13T14:41:21.166377+0000","last_clean":"2024-09-13T14:41:21.166377+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:41:21.166377+0000","last_undegraded":"2024-09-13T14:41:21.166377+0000","last_fullsized":"2024-09-13T14:41:21.166377+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965981+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:41:21.965981+0000","last_peered":"2024-09-13T14:41:21.965981+0000","last_clean":"2024-09-13T14:41:21.965981+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:41:21.965981+0000","last_undegraded":"2024-09-13T14:41:21.965981+0000","last_fullsized":"2024-09-13T14:41:21.965981+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820695+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:41:21.820695+0000","last_peered":"2024-09-13T14:41:21.820695+0000","last_clean":"2024-09-13T14:41:21.820695+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:41:21.820695+0000","last_undegraded":"2024-09-13T14:41:21.820695+0000","last_fullsized":"2024-09-13T14:41:21.820695+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166849+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:41:21.166849+0000","last_peered":"2024-09-13T14:41:21.166849+0000","last_clean":"2024-09-13T14:41:21.166849+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:41:21.166849+0000","last_undegraded":"2024-09-13T14:41:21.166849+0000","last_fullsized":"2024-09-13T14:41:21.166849+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815380+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:41:20.815380+0000","last_peered":"2024-09-13T14:41:20.815380+0000","last_clean":"2024-09-13T14:41:20.815380+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:41:20.815380+0000","last_undegraded":"2024-09-13T14:41:20.815380+0000","last_fullsized":"2024-09-13T14:41:20.815380+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166835+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:41:21.166835+0000","last_peered":"2024-09-13T14:41:21.166835+0000","last_clean":"2024-09-13T14:41:21.166835+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:41:21.166835+0000","last_undegraded":"2024-09-13T14:41:21.166835+0000","last_fullsized":"2024-09-13T14:41:21.166835+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819430+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:41:20.819430+0000","last_peered":"2024-09-13T14:41:20.819430+0000","last_clean":"2024-09-13T14:41:20.819430+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:41:20.819430+0000","last_undegraded":"2024-09-13T14:41:20.819430+0000","last_fullsized":"2024-09-13T14:41:20.819430+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166691+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:41:21.166691+0000","last_peered":"2024-09-13T14:41:21.166691+0000","last_clean":"2024-09-13T14:41:21.166691+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:41:21.166691+0000","last_undegraded":"2024-09-13T14:41:21.166691+0000","last_fullsized":"2024-09-13T14:41:21.166691+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818958+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:41:20.818958+0000","last_peered":"2024-09-13T14:41:20.818958+0000","last_clean":"2024-09-13T14:41:20.818958+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:41:20.818958+0000","last_undegraded":"2024-09-13T14:41:20.818958+0000","last_fullsized":"2024-09-13T14:41:20.818958+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815553+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:41:20.815553+0000","last_peered":"2024-09-13T14:41:20.815553+0000","last_clean":"2024-09-13T14:41:20.815553+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:41:20.815553+0000","last_undegraded":"2024-09-13T14:41:20.815553+0000","last_fullsized":"2024-09-13T14:41:20.815553+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.827083+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:41:21.827083+0000","last_peered":"2024-09-13T14:41:21.827083+0000","last_clean":"2024-09-13T14:41:21.827083+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:41:21.827083+0000","last_undegraded":"2024-09-13T14:41:21.827083+0000","last_fullsized":"2024-09-13T14:41:21.827083+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965604+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:41:21.965604+0000","last_peered":"2024-09-13T14:41:21.965604+0000","last_clean":"2024-09-13T14:41:21.965604+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:41:21.965604+0000","last_undegraded":"2024-09-13T14:41:21.965604+0000","last_fullsized":"2024-09-13T14:41:21.965604+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819134+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:41:20.819134+0000","last_peered":"2024-09-13T14:41:20.819134+0000","last_clean":"2024-09-13T14:41:20.819134+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:41:20.819134+0000","last_undegraded":"2024-09-13T14:41:20.819134+0000","last_fullsized":"2024-09-13T14:41:20.819134+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819005+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:41:20.819005+0000","last_peered":"2024-09-13T14:41:20.819005+0000","last_clean":"2024-09-13T14:41:20.819005+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:41:20.819005+0000","last_undegraded":"2024-09-13T14:41:20.819005+0000","last_fullsized":"2024-09-13T14:41:20.819005+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819521+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:41:20.819521+0000","last_peered":"2024-09-13T14:41:20.819521+0000","last_clean":"2024-09-13T14:41:20.819521+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:41:20.819521+0000","last_undegraded":"2024-09-13T14:41:20.819521+0000","last_fullsized":"2024-09-13T14:41:20.819521+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819323+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:41:20.819323+0000","last_peered":"2024-09-13T14:41:20.819323+0000","last_clean":"2024-09-13T14:41:20.819323+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:41:20.819323+0000","last_undegraded":"2024-09-13T14:41:20.819323+0000","last_fullsized":"2024-09-13T14:41:20.819323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965464+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:41:21.965464+0000","last_peered":"2024-09-13T14:41:21.965464+0000","last_clean":"2024-09-13T14:41:21.965464+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:41:21.965464+0000","last_undegraded":"2024-09-13T14:41:21.965464+0000","last_fullsized":"2024-09-13T14:41:21.965464+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821276+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:41:21.821276+0000","last_peered":"2024-09-13T14:41:21.821276+0000","last_clean":"2024-09-13T14:41:21.821276+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:41:21.821276+0000","last_undegraded":"2024-09-13T14:41:21.821276+0000","last_fullsized":"2024-09-13T14:41:21.821276+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":434,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166441+0000","last_change":"2024-09-13T14:34:44.816346+0000","last_active":"2024-09-13T14:41:21.166441+0000","last_peered":"2024-09-13T14:41:21.166441+0000","last_clean":"2024-09-13T14:41:21.166441+0000","last_became_active":"2024-09-13T14:34:44.815951+0000","last_became_peered":"2024-09-13T14:34:44.815951+0000","last_unstale":"2024-09-13T14:41:21.166441+0000","last_undegraded":"2024-09-13T14:41:21.166441+0000","last_fullsized":"2024-09-13T14:41:21.166441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:52:31.648689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815366+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:41:20.815366+0000","last_peered":"2024-09-13T14:41:20.815366+0000","last_clean":"2024-09-13T14:41:20.815366+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:41:20.815366+0000","last_undegraded":"2024-09-13T14:41:20.815366+0000","last_fullsized":"2024-09-13T14:41:20.815366+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965675+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:41:21.965675+0000","last_peered":"2024-09-13T14:41:21.965675+0000","last_clean":"2024-09-13T14:41:21.965675+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:41:21.965675+0000","last_undegraded":"2024-09-13T14:41:21.965675+0000","last_fullsized":"2024-09-13T14:41:21.965675+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821150+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:41:21.821150+0000","last_peered":"2024-09-13T14:41:21.821150+0000","last_clean":"2024-09-13T14:41:21.821150+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:41:21.821150+0000","last_undegraded":"2024-09-13T14:41:21.821150+0000","last_fullsized":"2024-09-13T14:41:21.821150+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965746+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:41:21.965746+0000","last_peered":"2024-09-13T14:41:21.965746+0000","last_clean":"2024-09-13T14:41:21.965746+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:41:21.965746+0000","last_undegraded":"2024-09-13T14:41:21.965746+0000","last_fullsized":"2024-09-13T14:41:21.965746+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822144+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:41:21.822144+0000","last_peered":"2024-09-13T14:41:21.822144+0000","last_clean":"2024-09-13T14:41:21.822144+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:41:21.822144+0000","last_undegraded":"2024-09-13T14:41:21.822144+0000","last_fullsized":"2024-09-13T14:41:21.822144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822800+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:41:21.822800+0000","last_peered":"2024-09-13T14:41:21.822800+0000","last_clean":"2024-09-13T14:41:21.822800+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:41:21.822800+0000","last_undegraded":"2024-09-13T14:41:21.822800+0000","last_fullsized":"2024-09-13T14:41:21.822800+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.818865+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:41:21.818865+0000","last_peered":"2024-09-13T14:41:21.818865+0000","last_clean":"2024-09-13T14:41:21.818865+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:41:21.818865+0000","last_undegraded":"2024-09-13T14:41:21.818865+0000","last_fullsized":"2024-09-13T14:41:21.818865+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820483+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:41:21.820483+0000","last_peered":"2024-09-13T14:41:21.820483+0000","last_clean":"2024-09-13T14:41:21.820483+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:41:21.820483+0000","last_undegraded":"2024-09-13T14:41:21.820483+0000","last_fullsized":"2024-09-13T14:41:21.820483+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820520+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:41:21.820520+0000","last_peered":"2024-09-13T14:41:21.820520+0000","last_clean":"2024-09-13T14:41:21.820520+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:41:21.820520+0000","last_undegraded":"2024-09-13T14:41:21.820520+0000","last_fullsized":"2024-09-13T14:41:21.820520+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815352+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:41:20.815352+0000","last_peered":"2024-09-13T14:41:20.815352+0000","last_clean":"2024-09-13T14:41:20.815352+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:41:20.815352+0000","last_undegraded":"2024-09-13T14:41:20.815352+0000","last_fullsized":"2024-09-13T14:41:20.815352+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965424+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:41:21.965424+0000","last_peered":"2024-09-13T14:41:21.965424+0000","last_clean":"2024-09-13T14:41:21.965424+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:41:21.965424+0000","last_undegraded":"2024-09-13T14:41:21.965424+0000","last_fullsized":"2024-09-13T14:41:21.965424+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820676+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:41:21.820676+0000","last_peered":"2024-09-13T14:41:21.820676+0000","last_clean":"2024-09-13T14:41:21.820676+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:41:21.820676+0000","last_undegraded":"2024-09-13T14:41:21.820676+0000","last_fullsized":"2024-09-13T14:41:21.820676+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966542+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:41:21.966542+0000","last_peered":"2024-09-13T14:41:21.966542+0000","last_clean":"2024-09-13T14:41:21.966542+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:41:21.966542+0000","last_undegraded":"2024-09-13T14:41:21.966542+0000","last_fullsized":"2024-09-13T14:41:21.966542+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820878+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:41:21.820878+0000","last_peered":"2024-09-13T14:41:21.820878+0000","last_clean":"2024-09-13T14:41:21.820878+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:41:21.820878+0000","last_undegraded":"2024-09-13T14:41:21.820878+0000","last_fullsized":"2024-09-13T14:41:21.820878+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967296+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:41:21.967296+0000","last_peered":"2024-09-13T14:41:21.967296+0000","last_clean":"2024-09-13T14:41:21.967296+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:41:21.967296+0000","last_undegraded":"2024-09-13T14:41:21.967296+0000","last_fullsized":"2024-09-13T14:41:21.967296+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820776+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:41:21.820776+0000","last_peered":"2024-09-13T14:41:21.820776+0000","last_clean":"2024-09-13T14:41:21.820776+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:41:21.820776+0000","last_undegraded":"2024-09-13T14:41:21.820776+0000","last_fullsized":"2024-09-13T14:41:21.820776+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166693+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:41:21.166693+0000","last_peered":"2024-09-13T14:41:21.166693+0000","last_clean":"2024-09-13T14:41:21.166693+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:41:21.166693+0000","last_undegraded":"2024-09-13T14:41:21.166693+0000","last_fullsized":"2024-09-13T14:41:21.166693+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822779+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:41:21.822779+0000","last_peered":"2024-09-13T14:41:21.822779+0000","last_clean":"2024-09-13T14:41:21.822779+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:41:21.822779+0000","last_undegraded":"2024-09-13T14:41:21.822779+0000","last_fullsized":"2024-09-13T14:41:21.822779+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966155+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:41:21.966155+0000","last_peered":"2024-09-13T14:41:21.966155+0000","last_clean":"2024-09-13T14:41:21.966155+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:41:21.966155+0000","last_undegraded":"2024-09-13T14:41:21.966155+0000","last_fullsized":"2024-09-13T14:41:21.966155+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821040+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:41:21.821040+0000","last_peered":"2024-09-13T14:41:21.821040+0000","last_clean":"2024-09-13T14:41:21.821040+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:41:21.821040+0000","last_undegraded":"2024-09-13T14:41:21.821040+0000","last_fullsized":"2024-09-13T14:41:21.821040+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965320+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:41:21.965320+0000","last_peered":"2024-09-13T14:41:21.965320+0000","last_clean":"2024-09-13T14:41:21.965320+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:41:21.965320+0000","last_undegraded":"2024-09-13T14:41:21.965320+0000","last_fullsized":"2024-09-13T14:41:21.965320+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823085+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:41:21.823085+0000","last_peered":"2024-09-13T14:41:21.823085+0000","last_clean":"2024-09-13T14:41:21.823085+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:41:21.823085+0000","last_undegraded":"2024-09-13T14:41:21.823085+0000","last_fullsized":"2024-09-13T14:41:21.823085+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818297+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:41:20.818297+0000","last_peered":"2024-09-13T14:41:20.818297+0000","last_clean":"2024-09-13T14:41:20.818297+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:41:20.818297+0000","last_undegraded":"2024-09-13T14:41:20.818297+0000","last_fullsized":"2024-09-13T14:41:20.818297+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823090+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:41:21.823090+0000","last_peered":"2024-09-13T14:41:21.823090+0000","last_clean":"2024-09-13T14:41:21.823090+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:41:21.823090+0000","last_undegraded":"2024-09-13T14:41:21.823090+0000","last_fullsized":"2024-09-13T14:41:21.823090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819385+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:41:20.819385+0000","last_peered":"2024-09-13T14:41:20.819385+0000","last_clean":"2024-09-13T14:41:20.819385+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:41:20.819385+0000","last_undegraded":"2024-09-13T14:41:20.819385+0000","last_fullsized":"2024-09-13T14:41:20.819385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819252+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:41:20.819252+0000","last_peered":"2024-09-13T14:41:20.819252+0000","last_clean":"2024-09-13T14:41:20.819252+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:41:20.819252+0000","last_undegraded":"2024-09-13T14:41:20.819252+0000","last_fullsized":"2024-09-13T14:41:20.819252+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823984+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:41:21.823984+0000","last_peered":"2024-09-13T14:41:21.823984+0000","last_clean":"2024-09-13T14:41:21.823984+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:41:21.823984+0000","last_undegraded":"2024-09-13T14:41:21.823984+0000","last_fullsized":"2024-09-13T14:41:21.823984+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":37,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819034+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:41:21.819034+0000","last_peered":"2024-09-13T14:41:21.819034+0000","last_clean":"2024-09-13T14:41:21.819034+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:41:21.819034+0000","last_undegraded":"2024-09-13T14:41:21.819034+0000","last_fullsized":"2024-09-13T14:41:21.819034+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815522+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:41:20.815522+0000","last_peered":"2024-09-13T14:41:20.815522+0000","last_clean":"2024-09-13T14:41:20.815522+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:41:20.815522+0000","last_undegraded":"2024-09-13T14:41:20.815522+0000","last_fullsized":"2024-09-13T14:41:20.815522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965405+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:41:21.965405+0000","last_peered":"2024-09-13T14:41:21.965405+0000","last_clean":"2024-09-13T14:41:21.965405+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:41:21.965405+0000","last_undegraded":"2024-09-13T14:41:21.965405+0000","last_fullsized":"2024-09-13T14:41:21.965405+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.824143+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:41:21.824143+0000","last_peered":"2024-09-13T14:41:21.824143+0000","last_clean":"2024-09-13T14:41:21.824143+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:41:21.824143+0000","last_undegraded":"2024-09-13T14:41:21.824143+0000","last_fullsized":"2024-09-13T14:41:21.824143+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965209+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:41:21.965209+0000","last_peered":"2024-09-13T14:41:21.965209+0000","last_clean":"2024-09-13T14:41:21.965209+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:41:21.965209+0000","last_undegraded":"2024-09-13T14:41:21.965209+0000","last_fullsized":"2024-09-13T14:41:21.965209+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820975+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:41:21.820975+0000","last_peered":"2024-09-13T14:41:21.820975+0000","last_clean":"2024-09-13T14:41:21.820975+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:41:21.820975+0000","last_undegraded":"2024-09-13T14:41:21.820975+0000","last_fullsized":"2024-09-13T14:41:21.820975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965426+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:41:21.965426+0000","last_peered":"2024-09-13T14:41:21.965426+0000","last_clean":"2024-09-13T14:41:21.965426+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:41:21.965426+0000","last_undegraded":"2024-09-13T14:41:21.965426+0000","last_fullsized":"2024-09-13T14:41:21.965426+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822906+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:41:21.822906+0000","last_peered":"2024-09-13T14:41:21.822906+0000","last_clean":"2024-09-13T14:41:21.822906+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:41:21.822906+0000","last_undegraded":"2024-09-13T14:41:21.822906+0000","last_fullsized":"2024-09-13T14:41:21.822906+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820591+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:41:21.820591+0000","last_peered":"2024-09-13T14:41:21.820591+0000","last_clean":"2024-09-13T14:41:21.820591+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:41:21.820591+0000","last_undegraded":"2024-09-13T14:41:21.820591+0000","last_fullsized":"2024-09-13T14:41:21.820591+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965443+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:41:21.965443+0000","last_peered":"2024-09-13T14:41:21.965443+0000","last_clean":"2024-09-13T14:41:21.965443+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:41:21.965443+0000","last_undegraded":"2024-09-13T14:41:21.965443+0000","last_fullsized":"2024-09-13T14:41:21.965443+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965492+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:41:21.965492+0000","last_peered":"2024-09-13T14:41:21.965492+0000","last_clean":"2024-09-13T14:41:21.965492+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:41:21.965492+0000","last_undegraded":"2024-09-13T14:41:21.965492+0000","last_fullsized":"2024-09-13T14:41:21.965492+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819327+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:41:21.819327+0000","last_peered":"2024-09-13T14:41:21.819327+0000","last_clean":"2024-09-13T14:41:21.819327+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:41:21.819327+0000","last_undegraded":"2024-09-13T14:41:21.819327+0000","last_fullsized":"2024-09-13T14:41:21.819327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822891+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:41:21.822891+0000","last_peered":"2024-09-13T14:41:21.822891+0000","last_clean":"2024-09-13T14:41:21.822891+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:41:21.822891+0000","last_undegraded":"2024-09-13T14:41:21.822891+0000","last_fullsized":"2024-09-13T14:41:21.822891+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815497+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:41:20.815497+0000","last_peered":"2024-09-13T14:41:20.815497+0000","last_clean":"2024-09-13T14:41:20.815497+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:41:20.815497+0000","last_undegraded":"2024-09-13T14:41:20.815497+0000","last_fullsized":"2024-09-13T14:41:20.815497+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818881+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:41:20.818881+0000","last_peered":"2024-09-13T14:41:20.818881+0000","last_clean":"2024-09-13T14:41:20.818881+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:41:20.818881+0000","last_undegraded":"2024-09-13T14:41:20.818881+0000","last_fullsized":"2024-09-13T14:41:20.818881+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815647+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:41:20.815647+0000","last_peered":"2024-09-13T14:41:20.815647+0000","last_clean":"2024-09-13T14:41:20.815647+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:41:20.815647+0000","last_undegraded":"2024-09-13T14:41:20.815647+0000","last_fullsized":"2024-09-13T14:41:20.815647+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965755+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:41:21.965755+0000","last_peered":"2024-09-13T14:41:21.965755+0000","last_clean":"2024-09-13T14:41:21.965755+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:41:21.965755+0000","last_undegraded":"2024-09-13T14:41:21.965755+0000","last_fullsized":"2024-09-13T14:41:21.965755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965867+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:41:21.965867+0000","last_peered":"2024-09-13T14:41:21.965867+0000","last_clean":"2024-09-13T14:41:21.965867+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:41:21.965867+0000","last_undegraded":"2024-09-13T14:41:21.965867+0000","last_fullsized":"2024-09-13T14:41:21.965867+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967314+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:41:21.967314+0000","last_peered":"2024-09-13T14:41:21.967314+0000","last_clean":"2024-09-13T14:41:21.967314+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:41:21.967314+0000","last_undegraded":"2024-09-13T14:41:21.967314+0000","last_fullsized":"2024-09-13T14:41:21.967314+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":126,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822527+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:41:21.822527+0000","last_peered":"2024-09-13T14:41:21.822527+0000","last_clean":"2024-09-13T14:41:21.822527+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:41:21.822527+0000","last_undegraded":"2024-09-13T14:41:21.822527+0000","last_fullsized":"2024-09-13T14:41:21.822527+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234259,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492644,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":490494,"kb_avail":93252508,"statfs":{"total":95995035648,"available":95490568192,"internally_reserved":0,"allocated":2187264,"data_stored":1468787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":502266315},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495897,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428768,"kb_used_data":2132,"kb_used_omap":1,"kb_used_meta":426622,"kb_avail":93316384,"statfs":{"total":95995035648,"available":95555977216,"internally_reserved":0,"allocated":2183168,"data_stored":1468750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":436861383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757535,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588052,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":586494,"kb_avail":93157100,"statfs":{"total":95995035648,"available":95392870400,"internally_reserved":0,"allocated":1576960,"data_stored":873877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":600570310},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019171,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556064,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":554494,"kb_avail":93189088,"statfs":{"total":95995035648,"available":95425626112,"internally_reserved":0,"allocated":1589248,"data_stored":878405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":567802310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650384,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":85563,"kb_avail":93658032,"statfs":{"total":95995035648,"available":95905824768,"internally_reserved":0,"allocated":1572864,"data_stored":873639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":87617071},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879319,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76820,"kb_used_data":1544,"kb_used_omap":6,"kb_used_meta":75257,"kb_avail":93668332,"statfs":{"total":95995035648,"available":95916371968,"internally_reserved":0,"allocated":1581056,"data_stored":874255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":77063737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435335,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":201676,"kb_used_data":1532,"kb_used_omap":5,"kb_used_meta":200122,"kb_avail":93543476,"statfs":{"total":95995035648,"available":95788519424,"internally_reserved":0,"allocated":1568768,"data_stored":873056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":204925281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925910,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196116,"kb_used_data":2120,"kb_used_omap":5,"kb_used_meta":193978,"kb_avail":93549036,"statfs":{"total":95995035648,"available":95794212864,"internally_reserved":0,"allocated":2170880,"data_stored":1464521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":198633642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:41:47.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:41:47.415 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:47.694 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:47 smithi042 ceph-mon[108570]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:47.694 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:47 smithi042 ceph-mon[108570]: from='client.34734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:47.695 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:47 smithi042 ceph-mon[113926]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:47.695 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:47 smithi042 ceph-mon[113926]: from='client.34734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:47 smithi159 ceph-mon[102519]: pgmap v458: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:47 smithi159 ceph-mon[102519]: from='client.34734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:49.339 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:41:49.339 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:41:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:49 smithi042 ceph-mon[108570]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:49 smithi042 ceph-mon[113926]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:49 smithi159 ceph-mon[102519]: pgmap v459: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:49.929 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:49] "GET /metrics HTTP/1.1" 200 33981 "" "Prometheus/2.51.0" 2024-09-13T14:41:49.951 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":459,"stamp":"2024-09-13T14:41:48.163294+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1771228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2627260,"kb_used_data":14092,"kb_used_omap":28,"kb_used_meta":2613027,"kb_avail":747333956,"statfs":{"total":767960285184,"available":765269970944,"internally_reserved":0,"allocated":14430208,"data_stored":8775290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":2675740049},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002716"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965852+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:41:21.965852+0000","last_peered":"2024-09-13T14:41:21.965852+0000","last_clean":"2024-09-13T14:41:21.965852+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:41:21.965852+0000","last_undegraded":"2024-09-13T14:41:21.965852+0000","last_fullsized":"2024-09-13T14:41:21.965852+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":118,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166593+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:41:21.166593+0000","last_peered":"2024-09-13T14:41:21.166593+0000","last_clean":"2024-09-13T14:41:21.166593+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:41:21.166593+0000","last_undegraded":"2024-09-13T14:41:21.166593+0000","last_fullsized":"2024-09-13T14:41:21.166593+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":99,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815609+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:41:20.815609+0000","last_peered":"2024-09-13T14:41:20.815609+0000","last_clean":"2024-09-13T14:41:20.815609+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:41:20.815609+0000","last_undegraded":"2024-09-13T14:41:20.815609+0000","last_fullsized":"2024-09-13T14:41:20.815609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822140+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:41:21.822140+0000","last_peered":"2024-09-13T14:41:21.822140+0000","last_clean":"2024-09-13T14:41:21.822140+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:41:21.822140+0000","last_undegraded":"2024-09-13T14:41:21.822140+0000","last_fullsized":"2024-09-13T14:41:21.822140+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":121,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822997+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:41:21.822997+0000","last_peered":"2024-09-13T14:41:21.822997+0000","last_clean":"2024-09-13T14:41:21.822997+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:41:21.822997+0000","last_undegraded":"2024-09-13T14:41:21.822997+0000","last_fullsized":"2024-09-13T14:41:21.822997+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966277+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:41:21.966277+0000","last_peered":"2024-09-13T14:41:21.966277+0000","last_clean":"2024-09-13T14:41:21.966277+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:41:21.966277+0000","last_undegraded":"2024-09-13T14:41:21.966277+0000","last_fullsized":"2024-09-13T14:41:21.966277+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965981+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:41:21.965981+0000","last_peered":"2024-09-13T14:41:21.965981+0000","last_clean":"2024-09-13T14:41:21.965981+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:41:21.965981+0000","last_undegraded":"2024-09-13T14:41:21.965981+0000","last_fullsized":"2024-09-13T14:41:21.965981+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":110,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820695+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:41:21.820695+0000","last_peered":"2024-09-13T14:41:21.820695+0000","last_clean":"2024-09-13T14:41:21.820695+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:41:21.820695+0000","last_undegraded":"2024-09-13T14:41:21.820695+0000","last_fullsized":"2024-09-13T14:41:21.820695+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":103,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166849+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:41:21.166849+0000","last_peered":"2024-09-13T14:41:21.166849+0000","last_clean":"2024-09-13T14:41:21.166849+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:41:21.166849+0000","last_undegraded":"2024-09-13T14:41:21.166849+0000","last_fullsized":"2024-09-13T14:41:21.166849+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":85,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815380+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:41:20.815380+0000","last_peered":"2024-09-13T14:41:20.815380+0000","last_clean":"2024-09-13T14:41:20.815380+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:41:20.815380+0000","last_undegraded":"2024-09-13T14:41:20.815380+0000","last_fullsized":"2024-09-13T14:41:20.815380+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":104,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166835+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:41:21.166835+0000","last_peered":"2024-09-13T14:41:21.166835+0000","last_clean":"2024-09-13T14:41:21.166835+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:41:21.166835+0000","last_undegraded":"2024-09-13T14:41:21.166835+0000","last_fullsized":"2024-09-13T14:41:21.166835+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819430+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:41:20.819430+0000","last_peered":"2024-09-13T14:41:20.819430+0000","last_clean":"2024-09-13T14:41:20.819430+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:41:20.819430+0000","last_undegraded":"2024-09-13T14:41:20.819430+0000","last_fullsized":"2024-09-13T14:41:20.819430+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":88,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166691+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:41:21.166691+0000","last_peered":"2024-09-13T14:41:21.166691+0000","last_clean":"2024-09-13T14:41:21.166691+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:41:21.166691+0000","last_undegraded":"2024-09-13T14:41:21.166691+0000","last_fullsized":"2024-09-13T14:41:21.166691+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818958+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:41:20.818958+0000","last_peered":"2024-09-13T14:41:20.818958+0000","last_clean":"2024-09-13T14:41:20.818958+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:41:20.818958+0000","last_undegraded":"2024-09-13T14:41:20.818958+0000","last_fullsized":"2024-09-13T14:41:20.818958+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815553+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:41:20.815553+0000","last_peered":"2024-09-13T14:41:20.815553+0000","last_clean":"2024-09-13T14:41:20.815553+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:41:20.815553+0000","last_undegraded":"2024-09-13T14:41:20.815553+0000","last_fullsized":"2024-09-13T14:41:20.815553+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.827083+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:41:21.827083+0000","last_peered":"2024-09-13T14:41:21.827083+0000","last_clean":"2024-09-13T14:41:21.827083+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:41:21.827083+0000","last_undegraded":"2024-09-13T14:41:21.827083+0000","last_fullsized":"2024-09-13T14:41:21.827083+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965604+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:41:21.965604+0000","last_peered":"2024-09-13T14:41:21.965604+0000","last_clean":"2024-09-13T14:41:21.965604+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:41:21.965604+0000","last_undegraded":"2024-09-13T14:41:21.965604+0000","last_fullsized":"2024-09-13T14:41:21.965604+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":101,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819134+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:41:20.819134+0000","last_peered":"2024-09-13T14:41:20.819134+0000","last_clean":"2024-09-13T14:41:20.819134+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:41:20.819134+0000","last_undegraded":"2024-09-13T14:41:20.819134+0000","last_fullsized":"2024-09-13T14:41:20.819134+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819005+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:41:20.819005+0000","last_peered":"2024-09-13T14:41:20.819005+0000","last_clean":"2024-09-13T14:41:20.819005+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:41:20.819005+0000","last_undegraded":"2024-09-13T14:41:20.819005+0000","last_fullsized":"2024-09-13T14:41:20.819005+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819521+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:41:20.819521+0000","last_peered":"2024-09-13T14:41:20.819521+0000","last_clean":"2024-09-13T14:41:20.819521+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:41:20.819521+0000","last_undegraded":"2024-09-13T14:41:20.819521+0000","last_fullsized":"2024-09-13T14:41:20.819521+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819323+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:41:20.819323+0000","last_peered":"2024-09-13T14:41:20.819323+0000","last_clean":"2024-09-13T14:41:20.819323+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:41:20.819323+0000","last_undegraded":"2024-09-13T14:41:20.819323+0000","last_fullsized":"2024-09-13T14:41:20.819323+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":94,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965464+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:41:21.965464+0000","last_peered":"2024-09-13T14:41:21.965464+0000","last_clean":"2024-09-13T14:41:21.965464+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:41:21.965464+0000","last_undegraded":"2024-09-13T14:41:21.965464+0000","last_fullsized":"2024-09-13T14:41:21.965464+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821276+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:41:21.821276+0000","last_peered":"2024-09-13T14:41:21.821276+0000","last_clean":"2024-09-13T14:41:21.821276+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:41:21.821276+0000","last_undegraded":"2024-09-13T14:41:21.821276+0000","last_fullsized":"2024-09-13T14:41:21.821276+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":434,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166441+0000","last_change":"2024-09-13T14:34:44.816346+0000","last_active":"2024-09-13T14:41:21.166441+0000","last_peered":"2024-09-13T14:41:21.166441+0000","last_clean":"2024-09-13T14:41:21.166441+0000","last_became_active":"2024-09-13T14:34:44.815951+0000","last_became_peered":"2024-09-13T14:34:44.815951+0000","last_unstale":"2024-09-13T14:41:21.166441+0000","last_undegraded":"2024-09-13T14:41:21.166441+0000","last_fullsized":"2024-09-13T14:41:21.166441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:52:31.648689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815366+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:41:20.815366+0000","last_peered":"2024-09-13T14:41:20.815366+0000","last_clean":"2024-09-13T14:41:20.815366+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:41:20.815366+0000","last_undegraded":"2024-09-13T14:41:20.815366+0000","last_fullsized":"2024-09-13T14:41:20.815366+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965675+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:41:21.965675+0000","last_peered":"2024-09-13T14:41:21.965675+0000","last_clean":"2024-09-13T14:41:21.965675+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:41:21.965675+0000","last_undegraded":"2024-09-13T14:41:21.965675+0000","last_fullsized":"2024-09-13T14:41:21.965675+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821150+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:41:21.821150+0000","last_peered":"2024-09-13T14:41:21.821150+0000","last_clean":"2024-09-13T14:41:21.821150+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:41:21.821150+0000","last_undegraded":"2024-09-13T14:41:21.821150+0000","last_fullsized":"2024-09-13T14:41:21.821150+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":131,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965746+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:41:21.965746+0000","last_peered":"2024-09-13T14:41:21.965746+0000","last_clean":"2024-09-13T14:41:21.965746+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:41:21.965746+0000","last_undegraded":"2024-09-13T14:41:21.965746+0000","last_fullsized":"2024-09-13T14:41:21.965746+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":120,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822144+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:41:21.822144+0000","last_peered":"2024-09-13T14:41:21.822144+0000","last_clean":"2024-09-13T14:41:21.822144+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:41:21.822144+0000","last_undegraded":"2024-09-13T14:41:21.822144+0000","last_fullsized":"2024-09-13T14:41:21.822144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":133,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822800+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:41:21.822800+0000","last_peered":"2024-09-13T14:41:21.822800+0000","last_clean":"2024-09-13T14:41:21.822800+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:41:21.822800+0000","last_undegraded":"2024-09-13T14:41:21.822800+0000","last_fullsized":"2024-09-13T14:41:21.822800+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":116,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.818865+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:41:21.818865+0000","last_peered":"2024-09-13T14:41:21.818865+0000","last_clean":"2024-09-13T14:41:21.818865+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:41:21.818865+0000","last_undegraded":"2024-09-13T14:41:21.818865+0000","last_fullsized":"2024-09-13T14:41:21.818865+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820483+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:41:21.820483+0000","last_peered":"2024-09-13T14:41:21.820483+0000","last_clean":"2024-09-13T14:41:21.820483+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:41:21.820483+0000","last_undegraded":"2024-09-13T14:41:21.820483+0000","last_fullsized":"2024-09-13T14:41:21.820483+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820520+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:41:21.820520+0000","last_peered":"2024-09-13T14:41:21.820520+0000","last_clean":"2024-09-13T14:41:21.820520+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:41:21.820520+0000","last_undegraded":"2024-09-13T14:41:21.820520+0000","last_fullsized":"2024-09-13T14:41:21.820520+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":116,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815352+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:41:20.815352+0000","last_peered":"2024-09-13T14:41:20.815352+0000","last_clean":"2024-09-13T14:41:20.815352+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:41:20.815352+0000","last_undegraded":"2024-09-13T14:41:20.815352+0000","last_fullsized":"2024-09-13T14:41:20.815352+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965424+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:41:21.965424+0000","last_peered":"2024-09-13T14:41:21.965424+0000","last_clean":"2024-09-13T14:41:21.965424+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:41:21.965424+0000","last_undegraded":"2024-09-13T14:41:21.965424+0000","last_fullsized":"2024-09-13T14:41:21.965424+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820676+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:41:21.820676+0000","last_peered":"2024-09-13T14:41:21.820676+0000","last_clean":"2024-09-13T14:41:21.820676+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:41:21.820676+0000","last_undegraded":"2024-09-13T14:41:21.820676+0000","last_fullsized":"2024-09-13T14:41:21.820676+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966542+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:41:21.966542+0000","last_peered":"2024-09-13T14:41:21.966542+0000","last_clean":"2024-09-13T14:41:21.966542+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:41:21.966542+0000","last_undegraded":"2024-09-13T14:41:21.966542+0000","last_fullsized":"2024-09-13T14:41:21.966542+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":141,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820878+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:41:21.820878+0000","last_peered":"2024-09-13T14:41:21.820878+0000","last_clean":"2024-09-13T14:41:21.820878+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:41:21.820878+0000","last_undegraded":"2024-09-13T14:41:21.820878+0000","last_fullsized":"2024-09-13T14:41:21.820878+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967296+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:41:21.967296+0000","last_peered":"2024-09-13T14:41:21.967296+0000","last_clean":"2024-09-13T14:41:21.967296+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:41:21.967296+0000","last_undegraded":"2024-09-13T14:41:21.967296+0000","last_fullsized":"2024-09-13T14:41:21.967296+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820776+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:41:21.820776+0000","last_peered":"2024-09-13T14:41:21.820776+0000","last_clean":"2024-09-13T14:41:21.820776+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:41:21.820776+0000","last_undegraded":"2024-09-13T14:41:21.820776+0000","last_fullsized":"2024-09-13T14:41:21.820776+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.166693+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:41:21.166693+0000","last_peered":"2024-09-13T14:41:21.166693+0000","last_clean":"2024-09-13T14:41:21.166693+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:41:21.166693+0000","last_undegraded":"2024-09-13T14:41:21.166693+0000","last_fullsized":"2024-09-13T14:41:21.166693+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":111,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822779+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:41:21.822779+0000","last_peered":"2024-09-13T14:41:21.822779+0000","last_clean":"2024-09-13T14:41:21.822779+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:41:21.822779+0000","last_undegraded":"2024-09-13T14:41:21.822779+0000","last_fullsized":"2024-09-13T14:41:21.822779+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":80,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.966155+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:41:21.966155+0000","last_peered":"2024-09-13T14:41:21.966155+0000","last_clean":"2024-09-13T14:41:21.966155+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:41:21.966155+0000","last_undegraded":"2024-09-13T14:41:21.966155+0000","last_fullsized":"2024-09-13T14:41:21.966155+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":91,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.821040+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:41:21.821040+0000","last_peered":"2024-09-13T14:41:21.821040+0000","last_clean":"2024-09-13T14:41:21.821040+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:41:21.821040+0000","last_undegraded":"2024-09-13T14:41:21.821040+0000","last_fullsized":"2024-09-13T14:41:21.821040+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":77,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965320+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:41:21.965320+0000","last_peered":"2024-09-13T14:41:21.965320+0000","last_clean":"2024-09-13T14:41:21.965320+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:41:21.965320+0000","last_undegraded":"2024-09-13T14:41:21.965320+0000","last_fullsized":"2024-09-13T14:41:21.965320+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823085+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:41:21.823085+0000","last_peered":"2024-09-13T14:41:21.823085+0000","last_clean":"2024-09-13T14:41:21.823085+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:41:21.823085+0000","last_undegraded":"2024-09-13T14:41:21.823085+0000","last_fullsized":"2024-09-13T14:41:21.823085+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818297+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:41:20.818297+0000","last_peered":"2024-09-13T14:41:20.818297+0000","last_clean":"2024-09-13T14:41:20.818297+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:41:20.818297+0000","last_undegraded":"2024-09-13T14:41:20.818297+0000","last_fullsized":"2024-09-13T14:41:20.818297+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823090+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:41:21.823090+0000","last_peered":"2024-09-13T14:41:21.823090+0000","last_clean":"2024-09-13T14:41:21.823090+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:41:21.823090+0000","last_undegraded":"2024-09-13T14:41:21.823090+0000","last_fullsized":"2024-09-13T14:41:21.823090+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819385+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:41:20.819385+0000","last_peered":"2024-09-13T14:41:20.819385+0000","last_clean":"2024-09-13T14:41:20.819385+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:41:20.819385+0000","last_undegraded":"2024-09-13T14:41:20.819385+0000","last_fullsized":"2024-09-13T14:41:20.819385+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":109,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819252+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:41:20.819252+0000","last_peered":"2024-09-13T14:41:20.819252+0000","last_clean":"2024-09-13T14:41:20.819252+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:41:20.819252+0000","last_undegraded":"2024-09-13T14:41:20.819252+0000","last_fullsized":"2024-09-13T14:41:20.819252+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.823984+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:41:21.823984+0000","last_peered":"2024-09-13T14:41:21.823984+0000","last_clean":"2024-09-13T14:41:21.823984+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:41:21.823984+0000","last_undegraded":"2024-09-13T14:41:21.823984+0000","last_fullsized":"2024-09-13T14:41:21.823984+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":37,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819034+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:41:21.819034+0000","last_peered":"2024-09-13T14:41:21.819034+0000","last_clean":"2024-09-13T14:41:21.819034+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:41:21.819034+0000","last_undegraded":"2024-09-13T14:41:21.819034+0000","last_fullsized":"2024-09-13T14:41:21.819034+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":100,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815522+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:41:20.815522+0000","last_peered":"2024-09-13T14:41:20.815522+0000","last_clean":"2024-09-13T14:41:20.815522+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:41:20.815522+0000","last_undegraded":"2024-09-13T14:41:20.815522+0000","last_fullsized":"2024-09-13T14:41:20.815522+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965405+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:41:21.965405+0000","last_peered":"2024-09-13T14:41:21.965405+0000","last_clean":"2024-09-13T14:41:21.965405+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:41:21.965405+0000","last_undegraded":"2024-09-13T14:41:21.965405+0000","last_fullsized":"2024-09-13T14:41:21.965405+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.824143+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:41:21.824143+0000","last_peered":"2024-09-13T14:41:21.824143+0000","last_clean":"2024-09-13T14:41:21.824143+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:41:21.824143+0000","last_undegraded":"2024-09-13T14:41:21.824143+0000","last_fullsized":"2024-09-13T14:41:21.824143+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965209+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:41:21.965209+0000","last_peered":"2024-09-13T14:41:21.965209+0000","last_clean":"2024-09-13T14:41:21.965209+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:41:21.965209+0000","last_undegraded":"2024-09-13T14:41:21.965209+0000","last_fullsized":"2024-09-13T14:41:21.965209+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820975+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:41:21.820975+0000","last_peered":"2024-09-13T14:41:21.820975+0000","last_clean":"2024-09-13T14:41:21.820975+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:41:21.820975+0000","last_undegraded":"2024-09-13T14:41:21.820975+0000","last_fullsized":"2024-09-13T14:41:21.820975+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965426+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:41:21.965426+0000","last_peered":"2024-09-13T14:41:21.965426+0000","last_clean":"2024-09-13T14:41:21.965426+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:41:21.965426+0000","last_undegraded":"2024-09-13T14:41:21.965426+0000","last_fullsized":"2024-09-13T14:41:21.965426+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":122,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822906+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:41:21.822906+0000","last_peered":"2024-09-13T14:41:21.822906+0000","last_clean":"2024-09-13T14:41:21.822906+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:41:21.822906+0000","last_undegraded":"2024-09-13T14:41:21.822906+0000","last_fullsized":"2024-09-13T14:41:21.822906+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.820591+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:41:21.820591+0000","last_peered":"2024-09-13T14:41:21.820591+0000","last_clean":"2024-09-13T14:41:21.820591+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:41:21.820591+0000","last_undegraded":"2024-09-13T14:41:21.820591+0000","last_fullsized":"2024-09-13T14:41:21.820591+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":92,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965443+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:41:21.965443+0000","last_peered":"2024-09-13T14:41:21.965443+0000","last_clean":"2024-09-13T14:41:21.965443+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:41:21.965443+0000","last_undegraded":"2024-09-13T14:41:21.965443+0000","last_fullsized":"2024-09-13T14:41:21.965443+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":16,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965492+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:41:21.965492+0000","last_peered":"2024-09-13T14:41:21.965492+0000","last_clean":"2024-09-13T14:41:21.965492+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:41:21.965492+0000","last_undegraded":"2024-09-13T14:41:21.965492+0000","last_fullsized":"2024-09-13T14:41:21.965492+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":117,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.819327+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:41:21.819327+0000","last_peered":"2024-09-13T14:41:21.819327+0000","last_clean":"2024-09-13T14:41:21.819327+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:41:21.819327+0000","last_undegraded":"2024-09-13T14:41:21.819327+0000","last_fullsized":"2024-09-13T14:41:21.819327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":125,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822891+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:41:21.822891+0000","last_peered":"2024-09-13T14:41:21.822891+0000","last_clean":"2024-09-13T14:41:21.822891+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:41:21.822891+0000","last_undegraded":"2024-09-13T14:41:21.822891+0000","last_fullsized":"2024-09-13T14:41:21.822891+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.819131+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:41:20.819131+0000","last_peered":"2024-09-13T14:41:20.819131+0000","last_clean":"2024-09-13T14:41:20.819131+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:41:20.819131+0000","last_undegraded":"2024-09-13T14:41:20.819131+0000","last_fullsized":"2024-09-13T14:41:20.819131+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":102,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815497+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:41:20.815497+0000","last_peered":"2024-09-13T14:41:20.815497+0000","last_clean":"2024-09-13T14:41:20.815497+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:41:20.815497+0000","last_undegraded":"2024-09-13T14:41:20.815497+0000","last_fullsized":"2024-09-13T14:41:20.815497+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.818881+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:41:20.818881+0000","last_peered":"2024-09-13T14:41:20.818881+0000","last_clean":"2024-09-13T14:41:20.818881+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:41:20.818881+0000","last_undegraded":"2024-09-13T14:41:20.818881+0000","last_fullsized":"2024-09-13T14:41:20.818881+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":87,"reported_epoch":136,"state":"active+clean","last_fresh":"2024-09-13T14:41:20.815647+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:41:20.815647+0000","last_peered":"2024-09-13T14:41:20.815647+0000","last_clean":"2024-09-13T14:41:20.815647+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:41:20.815647+0000","last_undegraded":"2024-09-13T14:41:20.815647+0000","last_fullsized":"2024-09-13T14:41:20.815647+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965755+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:41:21.965755+0000","last_peered":"2024-09-13T14:41:21.965755+0000","last_clean":"2024-09-13T14:41:21.965755+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:41:21.965755+0000","last_undegraded":"2024-09-13T14:41:21.965755+0000","last_fullsized":"2024-09-13T14:41:21.965755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":113,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.965867+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:41:21.965867+0000","last_peered":"2024-09-13T14:41:21.965867+0000","last_clean":"2024-09-13T14:41:21.965867+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:41:21.965867+0000","last_undegraded":"2024-09-13T14:41:21.965867+0000","last_fullsized":"2024-09-13T14:41:21.965867+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":11,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.967314+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:41:21.967314+0000","last_peered":"2024-09-13T14:41:21.967314+0000","last_clean":"2024-09-13T14:41:21.967314+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:41:21.967314+0000","last_undegraded":"2024-09-13T14:41:21.967314+0000","last_fullsized":"2024-09-13T14:41:21.967314+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":126,"reported_epoch":137,"state":"active+clean","last_fresh":"2024-09-13T14:41:21.822527+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:41:21.822527+0000","last_peered":"2024-09-13T14:41:21.822527+0000","last_clean":"2024-09-13T14:41:21.822527+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:41:21.822527+0000","last_undegraded":"2024-09-13T14:41:21.822527+0000","last_fullsized":"2024-09-13T14:41:21.822527+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234260,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":492644,"kb_used_data":2136,"kb_used_omap":1,"kb_used_meta":490494,"kb_avail":93252508,"statfs":{"total":95995035648,"available":95490568192,"internally_reserved":0,"allocated":2187264,"data_stored":1468787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":502266315},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495898,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428768,"kb_used_data":2132,"kb_used_omap":1,"kb_used_meta":426622,"kb_avail":93316384,"statfs":{"total":95995035648,"available":95555977216,"internally_reserved":0,"allocated":2183168,"data_stored":1468750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":436861383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757536,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588052,"kb_used_data":1540,"kb_used_omap":1,"kb_used_meta":586494,"kb_avail":93157100,"statfs":{"total":95995035648,"available":95392870400,"internally_reserved":0,"allocated":1576960,"data_stored":873877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":600570310},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019172,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556064,"kb_used_data":1552,"kb_used_omap":1,"kb_used_meta":554494,"kb_avail":93189088,"statfs":{"total":95995035648,"available":95425626112,"internally_reserved":0,"allocated":1589248,"data_stored":878405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":567802310},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650384,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87120,"kb_used_data":1536,"kb_used_omap":4,"kb_used_meta":85563,"kb_avail":93658032,"statfs":{"total":95995035648,"available":95905824768,"internally_reserved":0,"allocated":1572864,"data_stored":873639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":87617071},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879319,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76820,"kb_used_data":1544,"kb_used_omap":6,"kb_used_meta":75257,"kb_avail":93668332,"statfs":{"total":95995035648,"available":95916371968,"internally_reserved":0,"allocated":1581056,"data_stored":874255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":77063737},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435336,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":201676,"kb_used_data":1532,"kb_used_omap":5,"kb_used_meta":200122,"kb_avail":93543476,"statfs":{"total":95995035648,"available":95788519424,"internally_reserved":0,"allocated":1568768,"data_stored":873056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":204925281},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925910,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":196116,"kb_used_data":2120,"kb_used_omap":5,"kb_used_meta":193978,"kb_avail":93549036,"statfs":{"total":95995035648,"available":95794212864,"internally_reserved":0,"allocated":2170880,"data_stored":1464521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":198633642},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:41:49.954 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T14:41:49.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-13T14:41:50.166 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:50.563 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:50 smithi042 ceph-mon[108570]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:50.563 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:50 smithi042 ceph-mon[113926]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:50.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:50 smithi159 ceph-mon[102519]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:41:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:51 smithi042 ceph-mon[108570]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:51.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:51 smithi042 ceph-mon[113926]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:51.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:51 smithi159 ceph-mon[102519]: pgmap v460: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:41:52.252 INFO:teuthology.orchestra.run.smithi042.stdout:epoch 137 2024-09-13T14:41:52.252 INFO:teuthology.orchestra.run.smithi042.stdout:fsid 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T14:41:52.252 INFO:teuthology.orchestra.run.smithi042.stdout:created 2024-09-13T14:13:54.973587+0000 2024-09-13T14:41:52.252 INFO:teuthology.orchestra.run.smithi042.stdout:modified 2024-09-13T14:41:21.815605+0000 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:crush_version 22 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:full_ratio 0.95 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:backfillfull_ratio 0.9 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:nearfull_ratio 0.85 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:require_min_compat_client luminous 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:min_compat_client luminous 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:require_osd_release reef 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:stretch_mode_enabled false 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 65 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-13T14:41:52.253 INFO:teuthology.orchestra.run.smithi042.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 70 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 2.00 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:max_osd 8 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 up in weight 1 up_from 94 up_thru 136 down_at 91 last_clean_interval [8,90) [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232] [v2:172.21.15.42:6804/3277294232,v1:172.21.15.42:6805/3277294232] exists,up 0a40fd71-cbc3-4195-b082-c4f19f7f1dd4 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 up in weight 1 up_from 109 up_thru 136 down_at 106 last_clean_interval [13,105) [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102] [v2:172.21.15.42:6812/192065102,v1:172.21.15.42:6813/192065102] exists,up 50b16819-9756-4dd5-957c-a2968d7f22bb 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 up in weight 1 up_from 126 up_thru 134 down_at 123 last_clean_interval [18,122) [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758] [v2:172.21.15.42:6820/3631420758,v1:172.21.15.42:6821/3631420758] exists,up de5587ea-c587-4c38-a8ce-6f2bf7f41b2d 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 up in weight 1 up_from 133 up_thru 134 down_at 128 last_clean_interval [25,127) [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048] [v2:172.21.15.42:6828/1641847048,v1:172.21.15.42:6829/1641847048] exists,up 0944a1a9-ea18-4457-8d5e-cf70f7e2be0f 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 up in weight 1 up_from 30 up_thru 136 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460] [v2:172.21.15.159:6802/3643931460,v1:172.21.15.159:6803/3643931460] exists,up 9cb88075-7a94-4890-8de9-a717d57eae24 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 up in weight 1 up_from 38 up_thru 133 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230] [v2:172.21.15.159:6810/1094422230,v1:172.21.15.159:6811/1094422230] exists,up 70e09ea8-aa92-41de-9d8b-f15ebee3edac 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 up in weight 1 up_from 46 up_thru 134 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602] [v2:172.21.15.159:6818/2578924602,v1:172.21.15.159:6819/2578924602] exists,up cc9e1f9f-78f5-452a-9f78-a47340fbdefe 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 up in weight 1 up_from 54 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602] [v2:172.21.15.159:6826/2913377602,v1:172.21.15.159:6827/2913377602] exists,up 85448d6e-e256-43f5-aa86-bf8c47b40863 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.3 [4,3] 2024-09-13T14:41:52.254 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.a [4,3] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.13 [4,1] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.15 [7,2] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.1a [4,3] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 3.1b [4,3] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 4.e [1,2] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 4.17 [1,2] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:pg_upmap_items 4.18 [1,5] 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/2982853112 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/1962929896 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/4078857182 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:6835/3376814479 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6800/3689509320 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6800/1382644857 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.255 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/3438146012 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/1529484914 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6800/4040829401 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/1729697261 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/879988818 expires 2024-09-14T14:14:27.026310+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6800/1561666977 expires 2024-09-14T14:14:27.026310+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/810688227 expires 2024-09-14T14:14:27.026310+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6801/1382644857 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/1269168698 expires 2024-09-14T14:14:50.147899+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/471235645 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6801/1561666977 expires 2024-09-14T14:14:27.026310+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:6834/3376814479 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/1816541453 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.256 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/196918433 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/240011015 expires 2024-09-14T14:14:27.026310+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/2764224015 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/800406081 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6801/3699801282 expires 2024-09-14T14:14:50.147899+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/2701547919 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/3181112294 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/3074293401 expires 2024-09-14T14:14:50.147899+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/713801628 expires 2024-09-14T14:14:50.147899+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/3559778733 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/2154994614 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6800/3699801282 expires 2024-09-14T14:14:50.147899+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/2845488129 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:0/1459960260 expires 2024-09-14T14:28:19.829303+0000 2024-09-13T14:41:52.257 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6801/3689509320 expires 2024-09-14T14:24:04.294604+0000 2024-09-13T14:41:52.258 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.159:0/2991823830 expires 2024-09-14T14:24:53.166700+0000 2024-09-13T14:41:52.258 INFO:teuthology.orchestra.run.smithi042.stdout:blocklist 172.21.15.42:6801/4040829401 expires 2024-09-14T14:19:24.781439+0000 2024-09-13T14:41:52.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3532374661' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:41:52.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3532374661' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:41:52.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3532374661' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-13T14:41:52.949 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 136 down_at 91 last_clean_interval [8,90) [v2:172.21.15.42:6802/3277294232,v1:172.21.15.42:6803/3277294232] [v2:172.21.15.42:6804/3277294232,v1:172.21.15.42:6805/3277294232] exists,up 0a40fd71-cbc3-4195-b082-c4f19f7f1dd4', 'osd.1 up in weight 1 up_from 109 up_thru 136 down_at 106 last_clean_interval [13,105) [v2:172.21.15.42:6810/192065102,v1:172.21.15.42:6811/192065102] [v2:172.21.15.42:6812/192065102,v1:172.21.15.42:6813/192065102] exists,up 50b16819-9756-4dd5-957c-a2968d7f22bb', 'osd.2 up in weight 1 up_from 126 up_thru 134 down_at 123 last_clean_interval [18,122) [v2:172.21.15.42:6818/3631420758,v1:172.21.15.42:6819/3631420758] [v2:172.21.15.42:6820/3631420758,v1:172.21.15.42:6821/3631420758] exists,up de5587ea-c587-4c38-a8ce-6f2bf7f41b2d', 'osd.3 up in weight 1 up_from 133 up_thru 134 down_at 128 last_clean_interval [25,127) [v2:172.21.15.42:6826/1641847048,v1:172.21.15.42:6827/1641847048] [v2:172.21.15.42:6828/1641847048,v1:172.21.15.42:6829/1641847048] exists,up 0944a1a9-ea18-4457-8d5e-cf70f7e2be0f', 'osd.4 up in weight 1 up_from 30 up_thru 136 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6800/3643931460,v1:172.21.15.159:6801/3643931460] [v2:172.21.15.159:6802/3643931460,v1:172.21.15.159:6803/3643931460] exists,up 9cb88075-7a94-4890-8de9-a717d57eae24', 'osd.5 up in weight 1 up_from 38 up_thru 133 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6808/1094422230,v1:172.21.15.159:6809/1094422230] [v2:172.21.15.159:6810/1094422230,v1:172.21.15.159:6811/1094422230] exists,up 70e09ea8-aa92-41de-9d8b-f15ebee3edac', 'osd.6 up in weight 1 up_from 46 up_thru 134 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6816/2578924602,v1:172.21.15.159:6817/2578924602] [v2:172.21.15.159:6818/2578924602,v1:172.21.15.159:6819/2578924602] exists,up cc9e1f9f-78f5-452a-9f78-a47340fbdefe', 'osd.7 up in weight 1 up_from 54 up_thru 129 down_at 0 last_clean_interval [0,0) [v2:172.21.15.159:6824/2913377602,v1:172.21.15.159:6825/2913377602] [v2:172.21.15.159:6826/2913377602,v1:172.21.15.159:6827/2913377602] exists,up 85448d6e-e256-43f5-aa86-bf8c47b40863'] 2024-09-13T14:41:52.950 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-13T14:41:53.156 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:53.565 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:53 smithi042 ceph-mon[108570]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:53.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:53 smithi042 ceph-mon[113926]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:53.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:53 smithi159 ceph-mon[102519]: pgmap v461: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:54.875 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "mon.a", 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster": "ceph", 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "admin_socket_mode": "", 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-13T14:41:54.876 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_client_required": "cephx, none", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_cluster_required": "cephx", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_debug": "true", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_service_required": "cephx", 2024-09-13T14:41:54.877 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "auth_supported": "", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_aio": "true", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_aio_poll_ms": "250", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_aio_reap_max": "16", 2024-09-13T14:41:54.878 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_async_discard_threads": "0", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_block_size": "4096", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_debug_aio": "false", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_debug_inflight_ios": "false", 2024-09-13T14:41:54.879 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_enable_discard": "false", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_flock_retry": "3", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_inject_crash": "0", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_ioring": "false", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_ioring_hipri": "false", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-13T14:41:54.880 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bdev_type": "", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_alloc_size": "1048576", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_allocator": "hybrid", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_buffered_io": "true", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_check_for_zeros": "false", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_compact_log_sync": "false", 2024-09-13T14:41:54.881 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_max_log_runway": "4194304", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_max_prefetch": "1048576", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_min_flush_size": "524288", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_min_log_runway": "1048576", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_replay_recovery": "false", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluefs_sync_write": "false", 2024-09-13T14:41:54.882 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_allocation_from_file": "true", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_allocator": "hybrid", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_async_db_compaction": "true", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bdev_label_multi": "true", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-13T14:41:54.883 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_create": "true", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_db_create": "false", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_db_path": "", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_db_size": "0", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_path": "", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_preallocate_file": "false", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_size": "107374182400", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_wal_create": "false", 2024-09-13T14:41:54.884 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_wal_path": "", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_block_wal_size": "100663296", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bluefs": "true", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_autotune": "true", 2024-09-13T14:41:54.885 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_size": "0", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-13T14:41:54.886 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cache_type": "2q", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_clone_cow": "true", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_mode": "none", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-13T14:41:54.887 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_csum_type": "crc32c", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_freelist": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-13T14:41:54.888 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_small_allocations": "0", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_default_buffered_read": "true", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_default_buffered_write": "false", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-13T14:41:54.889 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fail_eio": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_mount": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_umount": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-13T14:41:54.890 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_ignore_data_csum": "false", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_log_op_age": "5.000000", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_alloc_size": "0", 2024-09-13T14:41:54.891 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_blob_size": "0", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_max_deferred_txc": "32", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_min_alloc_size": "0", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_nid_prealloc": "1024", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-13T14:41:54.892 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_retry_disk_reads": "3", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_rocksdb_cf": "true", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-13T14:41:54.893 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_spdk_mem": "512", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_tracing": "false", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-13T14:41:54.894 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_write_v2": "false", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_write_v2_random": "false", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "bluestore_zero_block_detection": "false", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-13T14:41:54.895 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_cluster_require_signatures": "false", 2024-09-13T14:41:54.896 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_cluster_require_version": "2", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_require_signatures": "false", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_require_version": "2", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_service_require_signatures": "false", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_service_require_version": "2", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "cephx_sign_messages": "true", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "chdir": "", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "client_acl_type": "", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "client_asio_thread_count": "2", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "client_cache_mid": "0.750000", 2024-09-13T14:41:54.897 INFO:teuthology.orchestra.run.smithi042.stdout: "client_cache_size": "16384", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_caps_release_delay": "5", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_check_pool_perm": "true", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_debug_force_sync_read": "false", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_debug_getattr_caps": "false", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_debug_inject_features": "", 2024-09-13T14:41:54.898 INFO:teuthology.orchestra.run.smithi042.stdout: "client_debug_inject_tick_delay": "0", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_die_on_failed_remount": "false", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_dirsize_rbytes": "true", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_force_lazyio": "false", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_fs": "", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_inject_release_failure": "false", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_max_inline_size": "4096", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_mds_namespace": "", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_metadata": "", 2024-09-13T14:41:54.899 INFO:teuthology.orchestra.run.smithi042.stdout: "client_mount_gid": "-1", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_mount_timeout": "300", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_mount_uid": "-1", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_mountpoint": "/", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_notify_timeout": "10", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc": "true", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc_max_dirty": "104857600", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc_max_objects": "1000", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc_size": "209715200", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_oc_target_dirty": "8388608", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_permissions": "true", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_quota": "true", 2024-09-13T14:41:54.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_quota_df": "true", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_readahead_max_bytes": "0", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_readahead_max_periods": "4", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_readahead_min": "131072", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_reconnect_stale": "false", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_shutdown_timeout": "30", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_snapdir": ".snap", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_tick_interval": "1", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_trace": "", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_try_dentry_invalidate": "false", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_use_faked_inos": "false", 2024-09-13T14:41:54.901 INFO:teuthology.orchestra.run.smithi042.stdout: "client_use_random_mds": "false", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_graylog": "false", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_graylog_port": "12201", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_monitors": "default=true", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_syslog": "false", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "clog_to_syslog_level": "info", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster_addr": "-", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster_network": "", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster_network_interface": "", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "compressor_zlib_isal": "false", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "compressor_zlib_level": "5", 2024-09-13T14:41:54.902 INFO:teuthology.orchestra.run.smithi042.stdout: "compressor_zlib_winsize": "-15", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "compressor_zstd_level": "1", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_alien_op_num_threads": "6", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_osd_stat_interval": "0", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_seastar_cpu_cores": "", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crimson_seastar_num_threads": "0", 2024-09-13T14:41:54.903 INFO:teuthology.orchestra.run.smithi042.stdout: "crush_location": "", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "crush_location_hook": "", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "crush_location_hook_timeout": "10", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "daemonize": "false", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "daos_pool": "tank", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_alienstore": "0/5", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_allow_any_pool_priority": "false", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_asok": "1/5", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_asok_assert_abort": "false", 2024-09-13T14:41:54.904 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_asserts_on_shutdown": "true", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_auth": "1/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_bdev": "1/3", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_bluefs": "1/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_bluestore": "1/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_buffer": "0/1", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_ceph_dedup": "0/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_ceph_exporter": "1/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_cephfs_mirror": "0/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_cephsqlite": "0/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_client": "0/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_compressor": "1/5", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_context": "0/1", 2024-09-13T14:41:54.905 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_crimson_interrupt": "0/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_crush": "1/1", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_crypto": "1/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_cyanstore": "0/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_deliberately_leak_memory": "false", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_disable_randomized_ping": "false", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_dpdk": "1/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_eventtrace": "1/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_filer": "0/1", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_filestore": "1/3", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_finisher": "1/1", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_fuse": "1/5", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_heartbeat_testing_span": "0", 2024-09-13T14:41:54.906 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_heartbeatmap": "1/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_javaclient": "1/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_journal": "1/3", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_journaler": "0/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_kstore": "1/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_lockdep": "0/1", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mclock": "1/5", 2024-09-13T14:41:54.907 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_balancer": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_locker": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_log": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_log_expire": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_migrator": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mds_quiesce": "3/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_memstore": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mgr": "2/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mgrc": "1/5", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_mon": "20/20", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_monc": "0/10", 2024-09-13T14:41:54.908 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_ms": "1/1", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_none": "0/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_objclass": "0/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_objectcacher": "0/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_objecter": "0/1", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_optracker": "0/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_osd": "1/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_paxos": "20/20", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_perfcounter": "1/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_prioritycache": "1/5", 2024-09-13T14:41:54.909 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rados": "0/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rbd": "0/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rbd_mirror": "0/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rbd_pwl": "0/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rbd_replay": "0/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_refs": "0/0", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_reserver": "1/1", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw": "1/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_access": "1/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_datacache": "1/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_dbstore": "1/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_flight": "1/5", 2024-09-13T14:41:54.910 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_notification": "1/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rgw_sync": "1/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_rocksdb": "4/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore": "0/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_backref": "0/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_cache": "0/5", 2024-09-13T14:41:54.912 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_cleaner": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_device": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_epm": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_journal": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_lba": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_odata": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_omap": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_onode": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_t": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_seastore_tm": "0/5", 2024-09-13T14:41:54.913 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_striper": "0/1", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_test": "0/5", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_throttle": "1/1", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_timer": "0/1", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_tp": "0/5", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "debug_trace": "1/5", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "device_failure_prediction_mode": "none", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "err_to_graylog": "false", 2024-09-13T14:41:54.914 INFO:teuthology.orchestra.run.smithi042.stdout: "err_to_journald": "false", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "err_to_stderr": "true", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "err_to_syslog": "false", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "event_tracing": "false", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_addr": "0.0.0.0", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_cert_file": "", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_http_port": "9926", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_key_file": "", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_prio_limit": "5", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_sort_metrics": "true", 2024-09-13T14:41:54.915 INFO:teuthology.orchestra.run.smithi042.stdout: "exporter_stats_period": "5", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "fake_statfs_for_testing": "0", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "fatal_signal_handlers": "true", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filer_max_purge_ops": "10", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filer_max_truncate_ops": "128", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_apply_finisher_threads": "1", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_blackhole": "false", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_btrfs_clone_range": "true", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_btrfs_snap": "true", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_caller_concurrency": "10", 2024-09-13T14:41:54.916 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_collect_device_partition_information": "true", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_commit_timeout": "600.000000", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_debug_inject_read_err": "false", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_debug_omap_check": "false", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_debug_verify_split": "false", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_dump_file": "", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fadvise": "true", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fail_eio": "true", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fd_cache_shards": "16", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fd_cache_size": "128", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fiemap": "false", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fiemap_threshold": "4096", 2024-09-13T14:41:54.917 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_inject_stall": "0", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_journal_parallel": "false", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_journal_trailing": "false", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_journal_writeahead": "false", 2024-09-13T14:41:54.918 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_kill_at": "0", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattrs": "0", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-13T14:41:54.919 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_xattr_value_size": "0", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_merge_threshold": "-10", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_odsync_write": "false", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_omap_backend": "rocksdb", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_omap_backend_path": "", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-13T14:41:54.920 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_op_thread_timeout": "60", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_op_threads": "2", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_punch_hole": "false", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-13T14:41:54.921 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_queue_max_ops": "50", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_seek_data_hole": "false", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_sloppy_crc": "false", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_splice": "false", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_split_multiple": "2", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_split_rand_factor": "20", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_update_to": "1000", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-13T14:41:54.922 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_enable": "true", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-13T14:41:54.923 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_xfs_extsize": "false", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "filestore_zfs_snap": "false", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fio_dir": "/tmp/fio", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fsid": "4c2088c8-71da-11ef-bceb-c7b262605968", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_allow_other": "true", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_atomic_o_trunc": "true", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_big_writes": "true", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_debug": "false", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_default_permissions": "false", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_disable_pagecache": "false", 2024-09-13T14:41:54.924 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_max_write": "0", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_multithreaded": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_require_active_mds": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_set_user_groups": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_splice_move": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_splice_read": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_splice_write": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "fuse_use_invalidate_cb": "true", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "gss_target_name": "ceph", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "heartbeat_file": "", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "heartbeat_inject_failure": "0", 2024-09-13T14:41:54.925 INFO:teuthology.orchestra.run.smithi042.stdout: "heartbeat_interval": "5", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "host": "smithi042", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-13T14:41:54.926 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "inject_early_sigterm": "false", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "jaeger_agent_port": "6799", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "jaeger_tracing_enable": "false", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_aio": "true", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_align_min_size": "65536", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_block_align": "true", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_block_size": "4096", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_dio": "true", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_discard": "false", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_force_aio": "false", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_ignore_corruption": "false", 2024-09-13T14:41:54.927 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_max_write_bytes": "10485760", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_max_write_entries": "100", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_replay_from": "0", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_write_header_frequency": "0", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journal_zero_on_create": "false", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journaler_prefetch_periods": "10", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journaler_prezero_periods": "5", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "journaler_write_head_interval": "15", 2024-09-13T14:41:54.928 INFO:teuthology.orchestra.run.smithi042.stdout: "key": "", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "keyfile": "", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_backend": "rocksdb", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_default_stripe_size": "65536", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_fsck_on_mount": "false", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_max_bytes": "67108864", 2024-09-13T14:41:54.929 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_max_ops": "512", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_nid_prealloc": "1024", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_onode_map_size": "1024", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_sync_submit_transaction": "false", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "kstore_sync_transaction": "false", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "librados_thread_count": "2", 2024-09-13T14:41:54.930 INFO:teuthology.orchestra.run.smithi042.stdout: "lockdep": "false", 2024-09-13T14:41:54.931 INFO:teuthology.orchestra.run.smithi042.stdout: "lockdep_force_backtrace": "false", 2024-09-13T14:41:54.931 INFO:teuthology.orchestra.run.smithi042.stdout: "log_coarse_timestamps": "true", 2024-09-13T14:41:54.931 INFO:teuthology.orchestra.run.smithi042.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-13T14:41:54.931 INFO:teuthology.orchestra.run.smithi042.stdout: "log_flush_on_exit": "false", 2024-09-13T14:41:54.931 INFO:teuthology.orchestra.run.smithi042.stdout: "log_graylog_host": "127.0.0.1", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_graylog_port": "12201", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_max_new": "1000", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_max_recent": "10000", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_stderr_prefix": "", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_stop_at_utilization": "0.970000", 2024-09-13T14:41:54.932 INFO:teuthology.orchestra.run.smithi042.stdout: "log_to_file": "true", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "log_to_graylog": "false", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "log_to_journald": "false", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "log_to_stderr": "false", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "log_to_syslog": "false", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "max_rotating_auth_attempts": "10", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_action_on_write_error": "1", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_alternate_name_max": "8192", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_asio_thread_count": "2", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_export_pin": "true", 2024-09-13T14:41:54.933 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_fragment_dirs": "true", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_fragment_interval": "5", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_interval": "10", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_max": "-1", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_max_until": "-1", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_merge_size": "50", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_midchunk": "0.300000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_min_start": "0.200000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_minchunk": "0.001000", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_mode": "0", 2024-09-13T14:41:54.934 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_need_max": "1.200000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_need_min": "0.800000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_overload_epochs": "2", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_split_bits": "3", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_split_size": "10000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_target_decay": "10.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-13T14:41:54.935 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_beacon_grace": "15.000000", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_beacon_interval": "4.000000", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_mid": "0.700000", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_quiesce_delay": "0", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_release_free_interval": "10", 2024-09-13T14:41:54.936 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_reservation": "0.050000", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_trim_interval": "1", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cache_trim_threshold": "262144", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-13T14:41:54.937 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_client_prealloc_inos": "1000", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_connect_bootstrapping": "false", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_damage_table_max_entries": "10000", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_debug_auth_pins": "false", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_debug_frag": "false", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_debug_scatterstat": "false", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_debug_subtrees": "false", 2024-09-13T14:41:54.938 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_decay_halflife": "5.000000", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_default_dir_hash": "2", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_defer_session_stale": "true", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_deny_all_reconnect": "false", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dir_keys_per_op": "16384", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dir_max_commit_size": "10", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dir_max_entries": "0", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dir_prefetch": "true", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-13T14:41:54.939 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dump_cache_on_map": "false", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_early_reply": "true", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_enable_op_tracker": "true", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_enforce_unique_name": "true", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-13T14:41:54.940 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_export_ephemeral_random": "true", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-13T14:41:54.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_health_summarize_threshold": "10", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-13T14:41:54.942 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_health_dummy": "false", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_migrator_session_race": "false", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-13T14:41:54.943 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_join_fs": "", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_journal_format": "1", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_create_at": "0", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_dirfrag_at": "0", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_export_at": "0", 2024-09-13T14:41:54.944 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_import_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_journal_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_journal_expire_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_journal_replay_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_link_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_mdstable_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_openc_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_rename_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_kill_shutdown_at": "0", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_event_large_threshold": "524288", 2024-09-13T14:41:54.945 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_events_per_segment": "1024", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_max_events": "-1", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_max_segments": "128", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_pause": "false", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_segment_size": "0", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_trim_threshold": "128", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-13T14:41:54.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_log_warn_factor": "2.000000", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_caps_per_client": "1048576", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_completed_flushes": "100000", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_completed_requests": "100000", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_export_size": "20971520", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_file_recover": "32", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_purge_files": "64", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_purge_ops": "8192", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-13T14:41:54.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_max_snaps_per_dir": "100", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_metrics_update_interval": "2", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_min_caps_per_client": "100", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_min_caps_working_set": "10000", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_numa_node": "-1", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_complaint_time": "30.000000", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_history_duration": "600", 2024-09-13T14:41:54.948 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_history_size": "20", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_history_slow_op_size": "20", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_op_log_threshold": "5", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_ping_grace": "15", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_ping_interval": "5", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_max_caps": "30000", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-13T14:41:54.949 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_recall_warning_threshold": "262144", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_replay_interval": "1.000000", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_root_ino_gid": "0", 2024-09-13T14:41:54.950 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_root_ino_uid": "0", 2024-09-13T14:41:54.951 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-13T14:41:54.951 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-13T14:41:54.951 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-13T14:41:54.951 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-13T14:41:54.951 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-13T14:41:54.952 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_shutdown_check": "0", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_skip_ino": "0", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_snap_max_uid": "4294967294", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_snap_min_uid": "0", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_snap_rstat": "false", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_standby_replay_damaged": "false", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_symlink_recovery": "true", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_thrash_exports": "0", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_thrash_fragments": "0", 2024-09-13T14:41:54.953 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_tick_interval": "5.000000", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_valgrind_exit": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_verify_backtrace": "1", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_verify_scatter": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mds_wipe_sessions": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mempool_debug": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "memstore_device_bytes": "1073741824", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "memstore_page_set": "false", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "memstore_page_size": "65536", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_client_bytes": "134217728", 2024-09-13T14:41:54.954 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_client_messages": "512", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_disabled_modules": "", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_enable_op_tracker": "true", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_max_pg_creating": "1024", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_max_pg_num_change": "128", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_mds_bytes": "134217728", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_mds_messages": "128", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_mon_bytes": "134217728", 2024-09-13T14:41:54.955 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_mon_messages": "128", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_history_duration": "600", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_history_size": "20", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_op_log_threshold": "5", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_osd_bytes": "536870912", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_osd_messages": "8192", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_pool": "true", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-13T14:41:54.956 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_standby_modules": "true", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_stats_period": "5", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_stats_threshold": "5", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_tick_period": "2", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_allow_pool_delete": "true", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_allow_pool_size_one": "false", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_auth_validate_all_caps": "true", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_bytes": "104857600", 2024-09-13T14:41:54.957 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_directed_command_retry": "2", 2024-09-13T14:41:54.958 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-13T14:41:54.958 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-13T14:41:54.958 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-13T14:41:54.958 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-13T14:41:54.958 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_hunt_parallel": "3", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_log_interval": "1.000000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_ping_interval": "10.000000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_client_target_rank": "-1", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-13T14:41:54.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_level": "debug", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_file": "true", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_journald": "true", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_compact_on_bootstrap": "false", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_compact_on_start": "false", 2024-09-13T14:41:54.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_compact_on_trim": "true", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_cpu_threads": "4", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_crush_min_required_version": "hammer", 2024-09-13T14:41:54.961 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_daemon_bytes": "419430400", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_data_avail_crit": "5", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_data_avail_warn": "5", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_data_size_warn": "16106127360", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_dump_json": "false", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_dump_transactions": "false", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_extra_checks": "false", 2024-09-13T14:41:54.962 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_no_require_reef": "false", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_no_require_squid": "false", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_down_mkfs_grace": "60", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_down_uptime_grace": "60", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_election_default_strategy": "1", 2024-09-13T14:41:54.963 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_election_timeout": "5.000000", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_elector_ping_divisor": "2", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_enable_op_tracker": "true", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_fake_pool_delete": "false", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_force_quorum_join": "false", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_globalid_prealloc": "10000", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_detail_to_clog": "true", 2024-09-13T14:41:54.964 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_log_update_period": "5", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_max_detail": "50", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_to_clog": "true", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_to_clog_interval": "600", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_host": "[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] [v2:172.21.15.159:3300/0,v1:172.21.15.159:6789/0] [v2:172.21.15.42:3301/0,v1:172.21.15.42:6790/0]", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_host_override": "", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_initial_members": "", 2024-09-13T14:41:54.965 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_lease": "5.000000", 2024-09-13T14:41:54.966 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_log_full_interval": "50", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_log_max": "10000", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_log_max_summary": "50", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_log_epochs": "500", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_osd": "10000", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_pg_per_osd": "10000", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_pool_pg_num": "65536", 2024-09-13T14:41:54.967 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mds_force_trim_to": "0", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mds_skip_sanity": "true", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_memory_autotune": "true", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_memory_target": "2147483648", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_beacon_grace": "30", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_digest_period": "5", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_inactive_grace": "60", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_min_osdmap_epochs": "500", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-13T14:41:54.968 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_complaint_time": "30", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_history_duration": "600", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_history_size": "20", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_history_slow_op_size": "20", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_op_log_threshold": "5", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_auto_mark_in": "false", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-13T14:41:54.969 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_cache_size": "500", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_down_out_interval": "600", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_force_trim_to": "0", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-13T14:41:54.970 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_min_down_reporters": "2", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_report_timeout": "900", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-13T14:41:54.971 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-13T14:41:54.972 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_stuck_threshold": "60", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_probe_timeout": "2.000000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_reweight_max_change": "0.050000", 2024-09-13T14:41:54.973 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_reweight_max_osds": "4", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_scrub_interval": "86400", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_scrub_max_keys": "100", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_scrub_timeout": "300", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_session_timeout": "300", 2024-09-13T14:41:54.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_smart_report_timeout": "5", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_stat_smooth_intervals": "6", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_stretch_pool_min_size": "2", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_stretch_pool_size": "4", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-13T14:41:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_debug": "false", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_provider_kill_at": "0", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_requester_kill_at": "0", 2024-09-13T14:41:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_sync_timeout": "60.000000", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_target_pg_per_osd": "100", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_tick_interval": "5", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_timecheck_interval": "300.000000", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_use_min_delay_socket": "false", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_older_version_delay": "604800", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-13T14:41:54.977 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_misplaced": "false", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_older_version": "true", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-13T14:41:54.978 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "monmap": "", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-13T14:41:54.979 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "motr_tracing_enabled": "false", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_op_threads": "3", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_cm": "false", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_device_name": "", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_dscp": "96", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-13T14:41:54.980 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_local_gid": "", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_port_num": "1", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_sl": "3", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_support_srq": "true", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_rdma_type": "ib", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_async_reap_threshold": "5", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_before_connect": "false", 2024-09-13T14:41:54.981 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_ipv4": "true", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_ipv6": "false", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_msgr1": "true", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_msgr2": "true", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_port_max": "7568", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_port_min": "6800", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_retry_count": "3", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_bind_retry_delay": "5", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_blackhole_client": "false", 2024-09-13T14:41:54.982 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_blackhole_mds": "false", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_blackhole_mgr": "false", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_blackhole_mon": "false", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_blackhole_osd": "false", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_client_mode": "crc secure", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_cluster_mode": "crc secure", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_cluster_type": "", 2024-09-13T14:41:54.983 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_compress_secure": "false", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_connection_idle_timeout": "900", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_connection_ready_timeout": "10", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_crc_data": "true", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_crc_header": "true", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_die_on_bad_msg": "false", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_die_on_bug": "true", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_die_on_old_message": "true", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_die_on_skipped_message": "false", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_coremask": "0xF", 2024-09-13T14:41:54.984 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_enable_tso": "true", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_hugepages": "", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_lro": "true", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_memory_channel": "4", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-13T14:41:54.985 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_pmd": "", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_port_id": "0", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_dump_on_send": "false", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_initial_backoff": "0.200000", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_delay_max": "1.000000", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_delay_type": "", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_network_congestion": "0", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_inject_socket_failures": "0", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_learn_addr_from_peer": "true", 2024-09-13T14:41:54.986 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_max_accept_failures": "4", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_max_backoff": "15.000000", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_mon_client_mode": "secure crc", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_mon_service_mode": "secure crc", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_osd_compress_min_size": "1024", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_osd_compress_mode": "none", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_pq_min_cost": "65536", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_public_type": "", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_service_mode": "crc secure", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_tcp_listen_backlog": "512", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_tcp_nodelay": "true", 2024-09-13T14:41:54.987 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_tcp_rcvbuf": "0", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "ms_type": "async+posix", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "no_config_file": "false", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_completion_locks_per_session": "32", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_inflight_ops": "1024", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_tick_interval": "5.000000", 2024-09-13T14:41:54.988 INFO:teuthology.orchestra.run.smithi042.stdout: "objecter_timeout": "10.000000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "objectstore_blackhole": "false", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "openssl_engine_opts": "", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_delay_time": "5.000000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_hist_halflife": "1000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_max_low_ops": "2", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_max_ops": "4", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_agent_slop": "0.020000", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-13T14:41:54.989 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_asio_thread_count": "2", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_async_recovery_min_cost": "100", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backfill_scan_max": "512", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backfill_scan_min": "64", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backoff_on_degraded": "false", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backoff_on_peering": "false", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_backoff_on_unfound": "true", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_beacon_report_interval": "300", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_bench_duration": "30", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_bench_max_block_size": "67108864", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-13T14:41:54.990 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_blkin_trace_all": "false", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_check_for_log_corruption": "false", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_class_update_on_start": "true", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_client_message_cap": "256", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_client_message_size_cap": "524288000", 2024-09-13T14:41:54.991 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_client_op_priority": "63", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_client_watch_timeout": "30", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_command_max_records": "256", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_compact_on_start": "false", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_crush_update_on_start": "true", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_crush_update_weight_set": "true", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-13T14:41:54.992 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_feed_pullee": "-1", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_misdirected_ops": "false", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_no_acting_change": "false", 2024-09-13T14:41:54.993 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_no_purge_strays": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_op_order": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_shutdown": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-13T14:41:54.994 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_trim_objects": "false", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_verify_snaps": "false", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_keys": "1024", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_stride": "524288", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-13T14:41:54.995 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_default_notify_timeout": "30", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_delete_sleep": "0.000000", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_discard_disconnected_ops": "true", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_ec_partial_reads": "true", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_enable_op_tracker": "true", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-13T14:41:54.996 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_fast_info": "true", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_fast_shutdown": "true", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_function_tracing": "false", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_grace": "20", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_interval": "6", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-13T14:41:54.997 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_min_peers": "10", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_min_size": "2000", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_stale": "600", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_hit_set_max_size": "100000", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_hit_set_min_size": "1000", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-13T14:41:54.998 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_journal_size": "5120", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_kill_backfill_at": "0", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_map_cache_size": "50", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_map_dedup": "true", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_map_message_max": "40", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_map_share_max_epochs": "40", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_attr_name_len": "100", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_attr_size": "0", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_backfills": "1", 2024-09-13T14:41:54.999 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_markdown_count": "5", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_markdown_period": "600", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_object_name_len": "2048", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_object_namespace_len": "256", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_object_size": "134217728", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_pg_blocked_by": "16", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_pg_log_entries": "10000", 2024-09-13T14:41:55.000 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_pgls": "1024", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_push_cost": "8388608", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_push_objects": "10", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_scrubs": "3", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_trimming_pgs": "2", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_write_op_reply_len": "64", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_max_write_size": "90", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-13T14:41:55.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_profile": "balanced", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-13T14:41:55.002 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_base": "805306368", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_cache_min": "134217728", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_target": "4294967296", 2024-09-13T14:41:55.003 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_target_autotune": "false", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_min_pg_log_entries": "250", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_min_recovery_priority": "0", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-13T14:41:55.004 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mon_report_interval": "5", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_num_cache_shards": "32", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_num_op_tracker_shard": "32", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_numa_auto_affinity": "true", 2024-09-13T14:41:55.005 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_numa_node": "-1", 2024-09-13T14:41:55.006 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_numa_prefer_iface": "true", 2024-09-13T14:41:55.006 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-13T14:41:55.006 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_objecter_finishers": "1", 2024-09-13T14:41:55.006 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_objectstore": "bluestore", 2024-09-13T14:41:55.006 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_objectstore_fuse": "false", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_objectstore_tracing": "false", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_complaint_time": "30.000000", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_history_duration": "600", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_history_size": "20", 2024-09-13T14:41:55.007 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_history_slow_op_size": "20", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_log_threshold": "5", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_shards": "0", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_shards_hdd": "5", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_shards_ssd": "8", 2024-09-13T14:41:55.008 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_pq_min_cost": "65536", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-13T14:41:55.009 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_queue_cut_off": "high", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_op_thread_timeout": "15", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_open_classes_on_start": "true", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_os_flags": "0", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_peering_op_priority": "255", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_delete_cost": "1048576", 2024-09-13T14:41:55.010 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_delete_priority": "5", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_log_trim_max": "10000", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_log_trim_min": "100", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-13T14:41:55.011 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-13T14:41:55.012 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-13T14:41:55.012 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-13T14:41:55.012 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-13T14:41:55.012 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-13T14:41:55.012 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_crimson": "false", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-13T14:41:55.013 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-13T14:41:55.014 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_flags": "0", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_min_size": "0", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_pg_num": "32", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_pgp_num": "0", 2024-09-13T14:41:55.015 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_read_ratio": "70", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_size": "3", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_default_type": "replicated", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-13T14:41:55.016 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_push_per_object_cost": "1000", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recover_clone_overlap": "true", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_cost": "20971520", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_active": "0", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_max_single_start": "1", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_op_priority": "3", 2024-09-13T14:41:55.017 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_priority": "5", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_sleep": "0.000000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_repair_during_recovery": "false", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_requested_scrub_priority": "120", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_auto_repair": "false", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-13T14:41:55.018 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_begin_hour": "0", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_begin_week_day": "0", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_chunk_max": "25", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_chunk_min": "5", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_cost": "52428800", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_during_recovery": "false", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_end_hour": "0", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_end_week_day": "0", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_event_cost": "4096", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_invalid_stats": "true", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-13T14:41:55.019 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_max_preemptions": "5", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_priority": "5", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_scrub_sleep": "0.000000", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_skip_data_digest": "false", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_smart_report_timeout": "5", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_cost": "1048576", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_priority": "5", 2024-09-13T14:41:55.020 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_target_transaction_size": "30", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-13T14:41:55.021 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_tracing": "false", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_use_stale_snap": "false", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "osdc_blkin_trace_all": "false", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_kill_at": "0", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_max_join_drift": "10", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_min": "500", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_min_wait": "0.050000", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_propose_interval": "1.000000", 2024-09-13T14:41:55.022 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_service_trim_max": "500", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_service_trim_min": "250", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_stash_full_interval": "25", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_trim_max": "500", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "paxos_trim_min": "250", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "perf": "true", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "pid_file": "", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "public_addr": "-", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "public_addrv": "", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "public_bind_addr": "-", 2024-09-13T14:41:55.023 INFO:teuthology.orchestra.run.smithi042.stdout: "public_network": "172.21.0.0/20", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "public_network_interface": "", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "qat_compressor_busy_polling": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "qat_compressor_enabled": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "qat_compressor_session_max_number": "256", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rados_mon_op_timeout": "0", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rados_osd_op_timeout": "0", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rados_tracing": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_atime_update_interval": "60", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_balance_parent_reads": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_balance_snap_reads": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_blkin_trace_all": "false", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-13T14:41:55.024 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache": "true", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_policy": "writearound", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_size": "33554432", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_clone_copy_on_read": "false", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_compression_hint": "none", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_concurrent_management_ops": "10", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-09-13T14:41:55.025 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_clone_format": "auto", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_data_pool": "", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_features": "61", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_format": "2", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_map_options": "", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_order": "22", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_pool": "rbd", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_stripe_count": "0", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_default_stripe_unit": "0", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-13T14:41:55.026 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_enable_alloc_hint": "true", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_io_scheduler": "simple", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_order": "24", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_pool": "", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_journal_splay_width": "4", 2024-09-13T14:41:55.027 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_localize_parent_reads": "false", 2024-09-13T14:41:55.028 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_localize_snap_reads": "false", 2024-09-13T14:41:55.028 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-13T14:41:55.028 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-13T14:41:55.028 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-13T14:41:55.029 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-13T14:41:55.030 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_mtime_update_interval": "60", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_non_blocking_aio": "true", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_op_thread_timeout": "60", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_op_threads": "1", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_parent_cache_enabled": "false", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-13T14:41:55.031 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_plugins": "", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_bps_burst": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_bps_limit": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_exclude_ops": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_iops_burst": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_iops_limit": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-13T14:41:55.032 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-13T14:41:55.033 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_read_from_replica_policy": "default", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-13T14:41:55.034 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_skip_partial_discard": "true", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_tracing": "false", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_validate_names": "true", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rbd_validate_pool": "true", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "restapi_base_url": "", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "restapi_log_level": "", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_acl_grants_max_num": "100", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_admin_entry": "admin", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-13T14:41:55.035 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_asio_assert_yielding": "false", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_backend_store": "rados", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_barbican_url": "", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_beast_enable_async": "true", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_counters_cache": "false", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-13T14:41:55.036 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_cache_enabled": "true", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_cache_expiry_interval": "900", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_cache_lru_size": "25000", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_config_store": "rados", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_content_length_compat": "false", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_copy_obj_progress": "true", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_cors_rules_max_num": "100", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_cross_domain_policy": "", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_addr": "", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-13T14:41:55.037 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_password": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_kmip_username": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_require_ssl": "true", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-13T14:41:55.038 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_addr": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_auth": "token", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_namespace": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_prefix": "", 2024-09-13T14:41:55.039 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_token_file": "", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_curl_buffersize": "524288", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_curl_low_speed_time": "300", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-13T14:41:55.040 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-13T14:41:55.041 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_log_changes_size": "1000", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_log_num_shards": "128", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_log_window": "30", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-13T14:41:55.042 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_disable_s3select": "false", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-13T14:41:55.043 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dns_name": "", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dns_s3website_name": "", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dynamic_resharding": "true", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-13T14:41:55.044 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_gc_threads": "true", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_lc_threads": "true", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_mdsearch": "true", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_ops_log": "false", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_quota_threads": "true", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_static_website": "false", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enable_usage_log": "false", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_enforce_swift_acls": "true", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_exit_timeout_secs": "120", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_expose_bucket": "false", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_extended_http_attrs": "", 2024-09-13T14:41:55.045 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_filter": "none", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_frontends": "beast port=7480", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_deferred": "50", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_objs": "32", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_gc_processor_period": "3600", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-13T14:41:55.046 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_init_timeout": "300", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_kafka_connection_idle": "300", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-13T14:41:55.047 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_domain": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_password": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_password_path": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_project": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_tenant": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_token": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_token_path": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_admin_user": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_barbican_domain": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_barbican_password": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_barbican_project": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_barbican_user": "", 2024-09-13T14:41:55.048 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_url": "", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_debug_interval": "-1", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_lock_max_time": "90", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_max_objs": "32", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_max_rules": "1000", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_max_worker": "3", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-13T14:41:55.049 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lc_thread_delay": "0", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_dnattr": "uid", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_searchfilter": "", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ldap_uri": "", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_log_http_headers": "", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-13T14:41:55.050 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_log_object_name_utc": "false", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_attr_name_len": "0", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_attr_size": "0", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_chunk_size": "4194304", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_listing_results": "1000", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_notify_retries": "10", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_put_param_size": "1048576", 2024-09-13T14:41:55.051 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_put_size": "5368709120", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_max_slo_entries": "1000", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_md_log_max_shards": "64", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_mp_lock_max_time": "600", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-13T14:41:55.052 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_max_gc": "300", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_num_async_rados_threads": "32", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_num_control_oids": "8", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_numa_node": "-1", 2024-09-13T14:41:55.053 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_objexp_chunk_size": "100", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_objexp_gc_interval": "600", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_op_thread_timeout": "600", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_op_tracing": "false", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_opa_token": "", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_opa_url": "", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_opa_verify_ssl": "true", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-13T14:41:55.054 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ops_log_rados": "false", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_ops_log_socket_path": "", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_period_push_interval": "2.000000", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_cache_lanes": "3", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_cache_partitions": "3", 2024-09-13T14:41:55.055 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_print_continue": "true", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_rados_tracing": "false", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_realm": "", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_realm_id": "", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_region": "", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-13T14:41:55.056 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_request_uri": "", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_batch_size": "64", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_max_aio": "128", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_num_logs": "16", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_reshard_thread_interval": "600", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_resolve_cname": "false", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_run_sync_thread": "true", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-13T14:41:55.057 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_scheduler_type": "throttler", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_script_uri": "", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_service_provider_name": "", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_client_id": "", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_client_secret": "", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_entry": "sts", 2024-09-13T14:41:55.058 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_key": "sts", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_min_session_duration": "900", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sts_token_introspection_url": "", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_account_in_url": "false", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_auth_entry": "auth", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_auth_url": "", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_custom_header": "", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_need_stats": "true", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_tenant_name": "", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_token_expiration": "86400", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_url": "", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_url_prefix": "swift", 2024-09-13T14:41:55.059 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_lease_period": "120", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-13T14:41:55.060 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_thread_pool_size": "512", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_comment": "", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_createby": "", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_encoding": "", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_flag": "false", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_origin": "", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_torrent_tracker": "", 2024-09-13T14:41:55.061 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_trust_forwarded_https": "false", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_usage_max_shards": "32", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_usage_max_user_shards": "1", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_use_opa_authz": "false", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_counters_cache": "false", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_max_buckets": "1000", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_policies_max_num": "100", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-13T14:41:55.062 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_user_unique_email": "true", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_verify_ssl": "true", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zone": "", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zone_id": "", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zonegroup": "", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zonegroup_id": "", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgwlc_auto_session_clear": "true", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_block_size": "4096", 2024-09-13T14:41:55.063 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_size": "536870912", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-13T14:41:55.064 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_index_type": "binary_search", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_partition_filters": "false", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_perf": "false", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "run_dir": "/var/run/ceph", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_block_create": "true", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_cache_lru_size": "67108864", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-13T14:41:55.065 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_default_max_object_size": "16777216", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_device_size": "53687091200", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_full_integrity_check": "false", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_journal_batch_capacity": "16", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_main_device_type": "SSD", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-13T14:41:55.066 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "seastore_segment_size": "67108864", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "set_keepcaps": "false", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "setgroup": "ceph", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "setuser": "ceph", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "setuser_match_path": "", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "thp": "false", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "threadpool_default_timeout": "60", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "throttler_perf_counter": "true", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "tmp_dir": "/tmp", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-13T14:41:55.067 INFO:teuthology.orchestra.run.smithi042.stdout: "uadk_compressor_enabled": "false", 2024-09-13T14:41:55.068 INFO:teuthology.orchestra.run.smithi042.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-13T14:41:55.068 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:41:55.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:55 smithi159 ceph-mon[102519]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:55.766 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:55 smithi042 ceph-mon[113926]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:55.766 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:55 smithi042 ceph-mon[108570]: pgmap v462: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:55.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-13T14:41:55.999 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:57.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:57 smithi042 ceph-mon[108570]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:57.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:57 smithi042 ceph-mon[113926]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:57.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:57 smithi159 ceph-mon[102519]: pgmap v463: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:57.814 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a: {} 2024-09-13T14:41:57.815 INFO:teuthology.orchestra.run.smithi042.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:41:57.824 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b: {} 2024-09-13T14:41:57.825 INFO:teuthology.orchestra.run.smithi042.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:41:57.833 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c: {} 2024-09-13T14:41:57.833 INFO:teuthology.orchestra.run.smithi042.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T14:41:58.506 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-13T14:41:58.507 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-13T14:41:58.507 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:41:58.507 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:41:58.507 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-13T14:41:58.508 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-13T14:41:58.508 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-13T14:41:58.508 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:41:58.508 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:41:58.509 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T14:41:58.509 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:41:58.509 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:41:58.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T14:41:58.509 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-13T14:41:58.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:58.510 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-13T14:41:58.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:41:58.510 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-13T14:41:58.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:41:58.511 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-13T14:41:58.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:41:58.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:41:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:58.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3758803164' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:41:58.753 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:58.769 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:58.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:58.813 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:58.826 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:41:58.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3758803164' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:41:58.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3758803164' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T14:41:58.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:58.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:58.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:58.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:59.021 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:41:59.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:41:59.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:41:59.173 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:41:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:41:59.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:59.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:41:59.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:59.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:41:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:59.529 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:59.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:41:59.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:59.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:41:59.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:41:59.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:41:59 smithi042 ceph-mon[108570]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:59.728 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:41:59 smithi042 ceph-mon[113926]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:41:59 smithi159 ceph-mon[102519]: pgmap v464: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:41:59.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:41:59.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:41:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:41:59.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:41:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:41:59.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:41:59.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.054 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:00.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:41:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:41:59] "GET /metrics HTTP/1.1" 200 33983 "" "Prometheus/2.51.0" 2024-09-13T14:42:00.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:00.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:00.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:00.308 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:00.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:00.426 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:00.426 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:00.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:00.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:00.561 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:00.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:00.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:00.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:00.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:00.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:00.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:00.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:00.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:01.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:01.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:01.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:01.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:01.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:01.573 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:01 smithi042 ceph-mon[113926]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:01.573 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:01 smithi042 ceph-mon[108570]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:01.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:01.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:01.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:01.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:01 smithi159 ceph-mon[102519]: pgmap v465: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:01.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:01.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:01.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:01.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:01.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:02.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.079 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:02.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.181 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:02.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:02.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:02.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:02.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:02.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:02.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:02.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:02.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:02.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:02.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:02.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:02.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:02.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:02.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:02.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:02.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:03.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:03.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:03.219 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:03.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:03.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:03.344 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:03.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:03.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:03.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:03.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:03.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:03.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:03.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:03.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:03 smithi042 ceph-mon[108570]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:03 smithi042 ceph-mon[113926]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:03.695 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:03.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:03.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:03 smithi159 ceph-mon[102519]: pgmap v466: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:03.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:03.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:03.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:03.947 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:03.972 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:03.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:03.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:04.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:04.098 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:04.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:04.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.221 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:04.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:04.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:04.349 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:04.451 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:04.462 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:42:04.476 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:04.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:04.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:04.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:04.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:04.724 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:04.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:04.826 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:42:04.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:04.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:04.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:04.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:05.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:05.105 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:05.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:05.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:05.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:05.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:05.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:05.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:05.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[113926]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[113926]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[108570]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:05.711 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:05.712 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:05 smithi042 ceph-mon[108570]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:42:05.713 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:05.735 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:05.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:05 smithi159 ceph-mon[102519]: pgmap v467: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:05 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:05 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/197952' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:42:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:05 smithi159 ceph-mon[102519]: osdmap e138: 8 total, 8 up, 8 in 2024-09-13T14:42:05.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:05.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:05.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:05.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:05.986 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:05.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:06.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:06.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:06.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:06.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:06.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:06.467 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:06.470 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:42:06.470 INFO:teuthology.orchestra.run.smithi042.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:42:06.470 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:42:06.490 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:06.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:06.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:07.144 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_4' created 2024-09-13T14:42:07.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:07.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:07.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1069875863' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:07.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:07.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1069875863' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:06 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:07.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:07.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:42:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:06 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1069875863' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:06 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-13T14:42:07.270 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:07.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:07.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:07.306 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:07.307 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:42:07.308 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:07.396 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:07.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:07.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:07.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:07.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:07.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:07.775 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:07.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:07.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[108570]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:42:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[108570]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:42:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2134545623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[113926]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:42:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[113926]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:42:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2134545623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:07.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:07.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:07.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:08.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:08.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:08.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:08.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:07 smithi159 ceph-mon[102519]: pgmap v469: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:07 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-13T14:42:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:07 smithi159 ceph-mon[102519]: osdmap e139: 8 total, 8 up, 8 in 2024-09-13T14:42:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2134545623' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:08.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:08.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:08.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:08.409 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:08.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:08.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:08.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:08.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:08.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:08.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-13T14:42:08.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:08.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:08.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:08.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:08.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:08.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:08.977 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:08.977 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:08.994 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:08 smithi042 ceph-mon[113926]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:42:08.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:08 smithi042 ceph-mon[108570]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:42:09.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.199 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:08 smithi159 ceph-mon[102519]: osdmap e140: 8 total, 8 up, 8 in 2024-09-13T14:42:09.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:09.324 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:09.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:09.451 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:09.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:09.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:09.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:09.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:42:09.633 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:09.668 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:09.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:09.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:09.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:09.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:09.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[113926]: pgmap v472: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[113926]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[108570]: pgmap v472: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:09.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-mon[108570]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:42:09.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:09] "GET /metrics HTTP/1.1" 200 34051 "" "Prometheus/2.51.0" 2024-09-13T14:42:09.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:42:09.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:09.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:09.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:09.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:10.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.211 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:09 smithi159 ceph-mon[102519]: pgmap v472: 89 pgs: 2 creating+peering, 14 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:09 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:42:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:09 smithi159 ceph-mon[102519]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T14:42:10.314 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:10.337 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:10.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:10.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:10.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:10.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:10.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:10.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:10.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:10.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:10.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:10.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:10.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:10.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:10.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:10.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:10.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:11.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:11.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:11.192 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:11.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:11.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:11.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:11.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:11.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:11.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:11.599 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:11.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:11.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:11.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:11.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:11.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:11.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:11.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:11.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:11.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:11.988 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:12.083 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:11 smithi042 ceph-mon[108570]: pgmap v474: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:12.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:11 smithi042 ceph-mon[113926]: pgmap v474: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:12.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.173 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:42:12.173 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:42:12.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:12.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:11 smithi159 ceph-mon[102519]: pgmap v474: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:12.334 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:12.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:12.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:12.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:12.487 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:12.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:12.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:12.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:12.694 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:12.717 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:12.739 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:12.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:12.841 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:12.865 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:12.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.866 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:12.967 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:12.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:12.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:13.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:13.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:13.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:13.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:13.372 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:13.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:13.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:13.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:13.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:13.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:13.623 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:13.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:13.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:13.750 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:13.787 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-13T14:42:13.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:13.876 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:13.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:13.978 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:14.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:14.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:14.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:13 smithi159 ceph-mon[102519]: pgmap v475: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:14.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:14.057 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:13 smithi042 ceph-mon[113926]: pgmap v475: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:14.057 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:14.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:13 smithi042 ceph-mon[108570]: pgmap v475: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:14.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:42:14.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:14.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:14.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:14.231 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:14.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:14.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.380 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:14.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:14.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:14.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:14.610 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:14.635 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:14.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:14.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:14.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:14.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:14.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:14.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:14.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:14.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:15.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:15.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:15.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:14 smithi042 ceph-mon[113926]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:42:15.025 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:15.025 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:14 smithi042 ceph-mon[108570]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:42:15.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:15.138 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:15.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:15.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3651259322' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:42:15.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:14 smithi159 ceph-mon[102519]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T14:42:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.264 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:15.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:15.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:15.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:15.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:15.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:15.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:15.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:15.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:15.803 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:42:15.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:15.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:15.891 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:15.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:15.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[113926]: pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4271360120' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[108570]: pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:16.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4271360120' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.007 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:16.120 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:16.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:16.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:15 smithi159 ceph-mon[102519]: pgmap v477: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:15 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:42:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4271360120' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:16.257 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:16.257 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:16.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:16.374 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-13T14:42:16.375 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-13T14:42:16.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:16.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:16.378 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:42:16.378 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:42:16.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:16.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:16.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:16.567 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:42:16.567 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:42:16.567 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_157727 2024-09-13T14:42:16.567 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:42:16.568 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:42:16.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:16.675 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:16.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:16.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:16.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:16.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[113926]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:16.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:16 smithi042 ceph-mon[108570]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:42:16.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:16.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:16.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:16.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:17.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:17.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:17.187 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:17.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:17.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:17.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:16 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:17.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:16 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:17.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:16 smithi159 ceph-mon[102519]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T14:42:17.331 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:17.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:17.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:17.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:17.524 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:17.567 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3515 3499 218.675 218.688 0.00416879 0.0045593 2024-09-13T14:42:17.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:17.659 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:17.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:17.793 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.795 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:17.895 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:17.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:17.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:18.016 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:17 smithi042 ceph-mon[113926]: pgmap v479: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:18.016 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:17 smithi042 ceph-mon[108570]: pgmap v479: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:18.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:18.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:42:18.076 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:18.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:18.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:18.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:17 smithi159 ceph-mon[102519]: pgmap v479: 89 pgs: 89 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T14:42:18.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:18.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:18.406 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:18.439 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:42:18.440 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:18.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:18.478 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:42:18.479 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:18.516 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:42:18.516 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:18.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:18.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:18.554 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:42:18.555 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:18.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 6610 6594 206.028 193.438 0.00559522 0.00484363 2024-09-13T14:42:18.591 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:42:18.591 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:18.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:18.627 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:42:18.628 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:18.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:18.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:18.659 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:42:18.659 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:18.696 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:42:18.696 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:18.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:18.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:18 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3388181939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:18.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:18 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3388181939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:18.896 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:18.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:18.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:18.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:19.040 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:19.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:19.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:19.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:19.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:18 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3388181939' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:19.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:19.349 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:19.397 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:42:19.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:19.493 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:19.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:19.568 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 9969 9953 207.325 209.938 0.00427911 0.00481442 2024-09-13T14:42:19.595 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:19.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:19.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:19.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:19.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:19.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:19.870 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:19.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:19.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:19.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:19.998 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:19 smithi042 ceph-mon[108570]: pgmap v480: 89 pgs: 89 active+clean; 29 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2024-09-13T14:42:19.998 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:19] "GET /metrics HTTP/1.1" 200 34051 "" "Prometheus/2.51.0" 2024-09-13T14:42:20.000 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:19 smithi042 ceph-mon[113926]: pgmap v480: 89 pgs: 89 active+clean; 29 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2024-09-13T14:42:20.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:20.015 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:20.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:20.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:20.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:20.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:20.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:20.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:19 smithi159 ceph-mon[102519]: pgmap v480: 89 pgs: 89 active+clean; 29 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s wr, 57 op/s 2024-09-13T14:42:20.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:20.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:20.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.478 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.568 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 13158 13142 205.317 199.312 0.00252176 0.00486324 2024-09-13T14:42:20.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:20.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:20.715 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:20.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.755 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:20.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:20.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:20.887 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:20.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:20.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:21.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:21.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:21.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:21.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.295 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:21.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:21.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:21.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:21.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.568 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 16168 16152 201.868 188.125 0.00747773 0.00494616 2024-09-13T14:42:21.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:21.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:21.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:21.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:21.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:21.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:21.846 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:21.948 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:21.984 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:21.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:22.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:42:22.085 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:21 smithi042 ceph-mon[108570]: pgmap v481: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-09-13T14:42:22.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:21 smithi042 ceph-mon[113926]: pgmap v481: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-09-13T14:42:22.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:22.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:22.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:22.228 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:21 smithi159 ceph-mon[102519]: pgmap v481: 89 pgs: 89 active+clean; 104 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 206 op/s 2024-09-13T14:42:22.269 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.371 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:22.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:22.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:22.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:22.568 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 18911 18895 196.794 171.438 0.00489373 0.00507546 2024-09-13T14:42:22.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:22.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:22.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:22.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:22.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:22.975 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:22.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:23.115 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:23.206 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:23.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:23.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.248 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:23.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.349 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:23.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:23.483 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:23.532 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:23.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 21643 21627 193.048 170.75 0.00698675 0.00517488 2024-09-13T14:42:23.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:23.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.673 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:23.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:23.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:23.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:23.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:23.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:23.949 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:23.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:23.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:24.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:24.073 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:23 smithi042 ceph-mon[108570]: pgmap v482: 89 pgs: 89 active+clean; 719 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:42:24.073 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:23 smithi042 ceph-mon[113926]: pgmap v482: 89 pgs: 89 active+clean; 719 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:42:24.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:24.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:24.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:24.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:23 smithi159 ceph-mon[102519]: pgmap v482: 89 pgs: 89 active+clean; 719 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T14:42:24.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:24.364 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:42:24.406 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:42:24.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:24.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:24.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:24.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 24147 24131 188.478 156.5 0.00549109 0.00530048 2024-09-13T14:42:24.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:24.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:24.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:24.746 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:24.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:24.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:24.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:24.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:24.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:25.021 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:25.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:25.163 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.198 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:25.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:25.299 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:25.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:25.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:25.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:25.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:25.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 26582 26566 184.444 152.188 0.00448596 0.00541112 2024-09-13T14:42:25.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:25.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:25.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:25.702 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:25.730 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:25.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:25.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:25.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:25.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:25.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:26.005 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:26.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:25 smithi042 ceph-mon[113926]: pgmap v483: 89 pgs: 89 active+clean; 1014 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:42:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:25 smithi042 ceph-mon[113926]: from='client.34803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:25 smithi042 ceph-mon[108570]: pgmap v483: 89 pgs: 89 active+clean; 1014 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:42:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:25 smithi042 ceph-mon[108570]: from='client.34803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:26.107 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:26.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:26.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:26.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:25 smithi159 ceph-mon[102519]: pgmap v483: 89 pgs: 89 active+clean; 1014 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-09-13T14:42:26.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:25 smithi159 ceph-mon[102519]: from='client.34803 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:42:26.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:26.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:26.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:26.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:26.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:26.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:26.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:26.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 28848 28832 180.161 141.625 0.00810444 0.00554413 2024-09-13T14:42:26.616 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243496 7.f 7.299d1c8f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.817793+0000", 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.79112640099999, 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99376885400000003, 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.617 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243496 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817793+0000", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817793+0000", 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817794+0000", 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.361e-06 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817799+0000", 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3789999999999999e-06 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817807+0000", 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1119999999999996e-06 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817817+0000", 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0268999999999999e-05 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824559+0000", 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067425480000000001 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824569+0000", 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1290000000000002e-06 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811551+0000", 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.986982103 2024-09-13T14:42:26.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811562+0000", 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0953e-05 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243501 7.9 7.dcb72759 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.817830+0000", 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.79108953500003, 2024-09-13T14:42:26.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99179507, 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243501 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817830+0000", 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817830+0000", 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817830+0000", 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.89e-07 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817833+0000", 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6230000000000001e-06 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817841+0000", 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2959999999999999e-06 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817847+0000", 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3729999999999998e-06 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824314+0000", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064668 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824327+0000", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2539e-05 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809614+0000", 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98528750600000004 2024-09-13T14:42:26.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809625+0000", 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0444e-05 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243509 7.f 7.cf2fc1cf (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.817861+0000", 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.79105783400001, 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99373667899999996, 2024-09-13T14:42:26.627 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243509 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817861+0000", 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817861+0000", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817862+0000", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6199999999999998e-07 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817864+0000", 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4549999999999998e-06 2024-09-13T14:42:26.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817871+0000", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8020000000000003e-06 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817876+0000", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3839999999999996e-06 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824602+0000", 2024-09-13T14:42:26.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067255839999999997 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824610+0000", 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1610000000000003e-06 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811588+0000", 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98697824499999998 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811598+0000", 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5859999999999994e-06 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243511 7.9 7.9499d079 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.817884+0000", 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.791035093, 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99177591899999995, 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.632 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243511 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817884+0000", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817884+0000", 2024-09-13T14:42:26.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817884+0000", 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8299999999999998e-07 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817886+0000", 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8169999999999999e-06 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817893+0000", 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7700000000000004e-06 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.817898+0000", 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2120000000000002e-06 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824355+0000", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006457071 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.824363+0000", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1300000000000001e-06 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809650+0000", 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98528689599999997 2024-09-13T14:42:26.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809660+0000", 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6399999999999992e-06 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243517 7.4 7.45e970d4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.828879+0000", 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.78004060000001, 2024-09-13T14:42:26.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98047894800000002, 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243517 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828879+0000", 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828879+0000", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828880+0000", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6199999999999999e-06 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828885+0000", 2024-09-13T14:42:26.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5979999999999999e-06 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828897+0000", 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1916000000000001e-05 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828906+0000", 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.02e-06 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828965+0000", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9234999999999998e-05 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.828978+0000", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3478999999999999e-05 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809344+0000", 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.980365351 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809358+0000", 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3729e-05 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243521 7.f 7.94c76f0f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.833145+0000", 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.77577439100003, 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97849003599999995, 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.642 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243521 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833145+0000", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833145+0000", 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833146+0000", 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.494e-06 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833153+0000", 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7009999999999999e-06 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833170+0000", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6636000000000001e-05 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833180+0000", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.061e-05 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833235+0000", 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4321999999999999e-05 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.833243+0000", 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2060000000000007e-06 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811625+0000", 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97838241999999997 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.811635+0000", 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6469999999999993e-06 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.647 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243522 7.9 7.1d0d0599 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.837958+0000", 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.77096089899999, 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97175080599999997, 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.648 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243522 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837958+0000", 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837958+0000", 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837959+0000", 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7899999999999997e-07 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837963+0000", 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9539999999999998e-06 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837974+0000", 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1456e-05 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.837983+0000", 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9439999999999994e-06 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.838041+0000", 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7216000000000002e-05 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.838048+0000", 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6699999999999994e-06 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809699+0000", 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97165079799999998 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809709+0000", 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9890000000000004e-06 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34212.0:243524 7.9 7.65d023c9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e94)", 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:34:43.843154+0000", 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 462.76576567799998, 2024-09-13T14:42:26.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96658903200000001, 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34212", 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3267122291", 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 243524 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843154+0000", 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843154+0000", 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843155+0000", 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.86e-06 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843167+0000", 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1156e-05 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843195+0000", 2024-09-13T14:42:26.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8807e-05 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843240+0000", 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4509999999999999e-05 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843342+0000", 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000102132 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:43.843350+0000", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6580000000000002e-06 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809733+0000", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96638377099999995 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:34:44.809743+0000", 2024-09-13T14:42:26.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1379999999999996e-06 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:53824 8.1 8:884e6b52:::benchmark_data_smithi042_130700_object53823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e106)", 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:36.142054+0000", 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 410.46686495500001, 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0030502859999999, 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.659 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 53824 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142054+0000", 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142054+0000", 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142056+0000", 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.063e-06 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142092+0000", 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5395000000000002e-05 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142105+0000", 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.376e-05 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142115+0000", 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9499999999999996e-06 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142220+0000", 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000104256 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142230+0000", 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0084e-05 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.135300+0000", 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99307034699999996 2024-09-13T14:42:26.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.136893+0000", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001593197 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.137049+0000", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000155384 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.141941+0000", 2024-09-13T14:42:26.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048918449999999997 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.144827+0000", 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002886296 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.144938+0000", 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000110708 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145105+0000", 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016700100000000001 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:53825 8.1 8:86211f52:::benchmark_data_smithi042_130700_object53824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e106)", 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:36.142138+0000", 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 410.46678135600001, 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0033230639999999, 2024-09-13T14:42:26.666 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 53825 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142138+0000", 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142138+0000", 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142138+0000", 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6100000000000001e-07 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142158+0000", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0040000000000001e-05 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142166+0000", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4100000000000002e-06 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142171+0000", 2024-09-13T14:42:26.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1479999999999997e-06 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142265+0000", 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4083e-05 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.142272+0000", 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4000000000000003e-06 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.139281+0000", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99700862599999995 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.139422+0000", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014103 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.139579+0000", 2024-09-13T14:42:26.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015692899999999999 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145123+0000", 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0055436399999999999 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145176+0000", 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3368999999999997e-05 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145305+0000", 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012845399999999999 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145461+0000", 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000156374 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:53837 8.1 8:883278e3:::benchmark_data_smithi042_130700_object53836:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e106)", 2024-09-13T14:42:26.673 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:36.155934+0000", 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 410.45298544600001, 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98978160900000001, 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 53837 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155934+0000", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155934+0000", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155934+0000", 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4799999999999998e-07 2024-09-13T14:42:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155951+0000", 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7079999999999999e-05 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155962+0000", 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0086e-05 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.155966+0000", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.741e-06 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.156008+0000", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1312e-05 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:36.156013+0000", 2024-09-13T14:42:26.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1279999999999999e-06 2024-09-13T14:42:26.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:26.679 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.679 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.141976+0000", 2024-09-13T14:42:26.679 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98596339 2024-09-13T14:42:26.679 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.679 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.142109+0000", 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132531 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.142268+0000", 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000159753 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145131+0000", 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028621860000000001 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145503+0000", 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00037267700000000002 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145593+0000", 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0122000000000002e-05 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:37.145715+0000", 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012195499999999999 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86333 8.1 8:8daf8689:::benchmark_data_smithi042_130700_object86332:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e110)", 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:58.841092+0000", 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 387.76782719300002, 2024-09-13T14:42:26.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1586049140000001, 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86333 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841092+0000", 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841092+0000", 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841094+0000", 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7519999999999999e-06 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841144+0000", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0535000000000003e-05 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841158+0000", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4141999999999999e-05 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841169+0000", 2024-09-13T14:42:26.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0309000000000001e-05 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841478+0000", 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00030936000000000001 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.841490+0000", 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1771e-05 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.970697+0000", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1292068479999999 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.970862+0000", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000165701 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.971035+0000", 2024-09-13T14:42:26.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017208499999999999 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.980038+0000", 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0090037069999999997 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.980058+0000", 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9901000000000001e-05 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.998046+0000", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017987408 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.998126+0000", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0838000000000001e-05 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:59.999697+0000", 2024-09-13T14:42:26.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001570557 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34323.0:89539 8.9 e111/102 8:9b23b7a7:::benchmark_data_smithi042_130700_object89538:head v 111'5549, mlcod=111'5546)", 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:03.155028+0000", 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 383.453891204, 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15903023999999999, 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.690 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155028+0000", 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155028+0000", 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.691 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155028+0000", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5200000000000002e-07 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155064+0000", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5228999999999998e-05 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155071+0000", 2024-09-13T14:42:26.692 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.678e-06 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155076+0000", 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7389999999999999e-06 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.157016+0000", 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019402530000000001 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.693 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.157038+0000", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1435000000000001e-05 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.313973+0000", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15693511800000001 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.314058+0000", 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5335999999999997e-05 2024-09-13T14:42:26.694 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:89541 8.1 8:808ff7ab:::benchmark_data_smithi042_130700_object89540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:03.155152+0000", 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 383.45376757700001, 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.175799908, 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.695 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89541 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155152+0000", 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155152+0000", 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155153+0000", 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2559999999999999e-06 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155175+0000", 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2120000000000002e-05 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155185+0000", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7040000000000006e-06 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.155190+0000", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6520000000000003e-06 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.307155+0000", 2024-09-13T14:42:26.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15196492 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.307638+0000", 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00048312700000000002 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.310781+0000", 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031425250000000002 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.699 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.323768+0000", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012987232 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.323817+0000", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8749999999999999e-05 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.328723+0000", 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0049057609999999998 2024-09-13T14:42:26.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.328956+0000", 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000233472 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:03.330952+0000", 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019953890000000002 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.701 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:113021 8.c 8:37b75068:::benchmark_data_smithi042_130700_object113020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:28.998343+0000", 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.61057602699998, 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17288719999999999, 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.702 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 113021 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998343+0000", 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998343+0000", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998345+0000", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1390000000000002e-06 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998376+0000", 2024-09-13T14:42:26.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0195000000000001e-05 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998392+0000", 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6305000000000001e-05 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998403+0000", 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0823e-05 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.705 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998473+0000", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0011000000000004e-05 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998621+0000", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014802700000000001 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:28.998784+0000", 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000163413 2024-09-13T14:42:26.706 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.006050+0000", 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072659539999999998 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.012958+0000", 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006907659 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.707 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.170947+0000", 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15798911900000001 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.171062+0000", 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011479700000000001 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.171230+0000", 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000168758 2024-09-13T14:42:26.708 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:113027 8.6 8:67b9a244:::benchmark_data_smithi042_130700_object113026:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:29.008762+0000", 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.60015681900001, 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.167877999, 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.709 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 113027 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008762+0000", 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.710 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008762+0000", 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008764+0000", 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9989999999999998e-06 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.711 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008793+0000", 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8521999999999999e-05 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008812+0000", 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9561000000000001e-05 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.008821+0000", 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6070000000000008e-06 2024-09-13T14:42:26.712 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010265+0000", 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014438490000000001 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010417+0000", 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015230899999999999 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.713 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010571+0000", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015386499999999999 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172259+0000", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.161688056 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172313+0000", 2024-09-13T14:42:26.714 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3832999999999998e-05 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176544+0000", 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042306339999999996 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176595+0000", 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1094999999999998e-05 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176640+0000", 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5668999999999999e-05 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:113028 8.c 8:35e3a556:::benchmark_data_smithi042_130700_object113027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:29.010792+0000", 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.59812689400002, 2024-09-13T14:42:26.716 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18219596199999999, 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 113028 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010792+0000", 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.717 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010792+0000", 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.718 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010794+0000", 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4190000000000001e-06 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010820+0000", 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6060000000000001e-05 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.719 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010831+0000", 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1202e-05 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.010838+0000", 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5300000000000002e-06 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.720 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.012990+0000", 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021525400000000001 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.013134+0000", 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014367899999999999 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:42:26.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167161+0000", 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15402754699999999 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.171362+0000", 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042003049999999997 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.722 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.175613+0000", 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042515290000000004 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.192883+0000", 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017270033000000001 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.723 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.192937+0000", 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4344e-05 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.192988+0000", 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0773999999999999e-05 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.724 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:113030 8.6 8:67bce846:::benchmark_data_smithi042_130700_object113029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:29.011023+0000", 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.59789666500001, 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16570399499999999, 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:42:26.725 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 113030 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011023+0000", 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.726 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.727 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011023+0000", 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011023+0000", 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6400000000000001e-07 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.728 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011043+0000", 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9881000000000002e-05 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011052+0000", 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2769999999999994e-06 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.729 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.011058+0000", 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0170000000000002e-06 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.168013+0000", 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.156954502 2024-09-13T14:42:26.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.168155+0000", 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000142328 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.168291+0000", 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013570100000000001 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172347+0000", 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040561709999999999 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172383+0000", 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6162999999999999e-05 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176648+0000", 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042645499999999998 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176689+0000", 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1579000000000003e-05 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.733 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.176727+0000", 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7061999999999999e-05 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.734 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.34323.0:113027 8.6 e111/102 ondisk, result = 0)", 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:29.014660+0000", 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.59425901700001, 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.157610519, 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.014660+0000", 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.735 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.014660+0000", 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.014661+0000", 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.15e-06 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.014666+0000", 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.135e-06 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167768+0000", 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.153101551 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.737 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167780+0000", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2386e-05 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172238+0000", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044572040000000002 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172250+0000", 2024-09-13T14:42:26.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1989e-05 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.172271+0000", 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1104e-05 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34323.0:113036 8.2 e111/102 8:48c6b804:::benchmark_data_smithi042_130700_object113035:head v 111'7009, mlcod=111'7007)", 2024-09-13T14:42:26.739 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:29.020661+0000", 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 357.58825869499998, 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16364746099999999, 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.020661+0000", 2024-09-13T14:42:26.740 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.020661+0000", 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.020662+0000", 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8220000000000001e-06 2024-09-13T14:42:26.741 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.020705+0000", 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2449e-05 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167783+0000", 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.14707847900000001 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167794+0000", 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0445000000000001e-05 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167893+0000", 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8939999999999998e-05 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.743 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.167918+0000", 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5803999999999999e-05 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.184224+0000", 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016305710000000001 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:42:26.744 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:29.184308+0000", 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3812000000000003e-05 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:42:26.745 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:42:26.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:26.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:26.910 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:26.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:27.012 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:27.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:27.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:27.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:27.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:27.289 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:27.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:27.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:27.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:27.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:27.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 31144 31128 176.827 143.5 0.00431848 0.00564943 2024-09-13T14:42:27.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:27.615 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:27.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:27.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:27.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:27.856 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:27.898 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:27.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:28.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:28.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:27 smithi042 ceph-mon[108570]: pgmap v484: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s 2024-09-13T14:42:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:27 smithi042 ceph-mon[113926]: pgmap v484: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s 2024-09-13T14:42:28.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:28.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:28.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:27 smithi159 ceph-mon[102519]: pgmap v484: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 118 MiB/s wr, 1.90k op/s 2024-09-13T14:42:28.278 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:28.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:28.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:28.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:28.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:28.569 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 33480 33464 174.257 146 0.00647079 0.0057322 2024-09-13T14:42:28.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:28.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:28.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:28.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:28.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:28.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:28.869 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:28.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:28.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:28.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:29.008 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:29.110 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:29.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:29.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:29.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:29.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:29.391 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:29.559 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:29.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 35711 35695 171.578 139.438 0.00538848 0.00582281 2024-09-13T14:42:29.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:29.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:29.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:29.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:29.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:29.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:29.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:29.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:30.069 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:30.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:29 smithi042 ceph-mon[108570]: pgmap v485: 89 pgs: 89 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:42:30.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:29 smithi042 ceph-mon[113926]: pgmap v485: 89 pgs: 89 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:42:30.102 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:29] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-09-13T14:42:30.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:30.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:30.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:29 smithi159 ceph-mon[102519]: pgmap v485: 89 pgs: 89 active+clean; 1.6 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T14:42:30.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:30.351 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:30.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:30.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:30.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:30.520 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:30.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:30.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 37911 37895 169.143 137.5 0.0126777 0.00590721 2024-09-13T14:42:30.622 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:30.661 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:30.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:30.763 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:30.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:30.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:30.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:30.932 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:30.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:31.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:31.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:31.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:31.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:31.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:31.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:31.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:31.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:31.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 39953 39937 166.375 127.625 0.00510913 0.006005 2024-09-13T14:42:31.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:31.631 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:31.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:31.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:31.738 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":483,"stamp":"2024-09-13T14:42:24.175921+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1063131109,"num_objects":16239,"num_object_clones":0,"num_object_copies":48717,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16239,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":16492,"num_write_kb":1040081,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":1771228,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16421,"ondisk_log_size":16421,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5457548,"kb_used_data":2762828,"kb_used_omap":28,"kb_used_meta":2694627,"kb_avail":744503668,"statfs":{"total":767960285184,"available":762371756032,"internally_reserved":0,"allocated":2829135872,"data_stored":2823299826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":2759298449},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,3,5,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1062535168,"num_objects":16213,"num_object_clones":0,"num_object_copies":48639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16213,"num_write_kb":1037632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006276"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802626+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:42:15.802626+0000","last_peered":"2024-09-13T14:42:15.802626+0000","last_clean":"2024-09-13T14:42:15.802626+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:42:15.802626+0000","last_undegraded":"2024-09-13T14:42:15.802626+0000","last_fullsized":"2024-09-13T14:42:15.802626+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":125,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342838+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:42:14.342838+0000","last_peered":"2024-09-13T14:42:14.342838+0000","last_clean":"2024-09-13T14:42:14.342838+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:42:14.342838+0000","last_undegraded":"2024-09-13T14:42:14.342838+0000","last_fullsized":"2024-09-13T14:42:14.342838+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":105,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342550+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:42:14.342550+0000","last_peered":"2024-09-13T14:42:14.342550+0000","last_clean":"2024-09-13T14:42:14.342550+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:42:14.342550+0000","last_undegraded":"2024-09-13T14:42:14.342550+0000","last_fullsized":"2024-09-13T14:42:14.342550+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":133,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873346+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:42:15.873346+0000","last_peered":"2024-09-13T14:42:15.873346+0000","last_clean":"2024-09-13T14:42:15.873346+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:42:15.873346+0000","last_undegraded":"2024-09-13T14:42:15.873346+0000","last_fullsized":"2024-09-13T14:42:15.873346+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":134,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873738+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:42:15.873738+0000","last_peered":"2024-09-13T14:42:15.873738+0000","last_clean":"2024-09-13T14:42:15.873738+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:42:15.873738+0000","last_undegraded":"2024-09-13T14:42:15.873738+0000","last_fullsized":"2024-09-13T14:42:15.873738+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.341932+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:42:14.341932+0000","last_peered":"2024-09-13T14:42:14.341932+0000","last_clean":"2024-09-13T14:42:14.341932+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:42:14.341932+0000","last_undegraded":"2024-09-13T14:42:14.341932+0000","last_fullsized":"2024-09-13T14:42:14.341932+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801715+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:42:15.801715+0000","last_peered":"2024-09-13T14:42:15.801715+0000","last_clean":"2024-09-13T14:42:15.801715+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:42:15.801715+0000","last_undegraded":"2024-09-13T14:42:15.801715+0000","last_fullsized":"2024-09-13T14:42:15.801715+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":116,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.789855+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:42:13.789855+0000","last_peered":"2024-09-13T14:42:13.789855+0000","last_clean":"2024-09-13T14:42:13.789855+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:42:13.789855+0000","last_undegraded":"2024-09-13T14:42:13.789855+0000","last_fullsized":"2024-09-13T14:42:13.789855+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"143'1302","reported_seq":1321,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:23.475084+0000","last_change":"2024-09-13T14:42:07.783733+0000","last_active":"2024-09-13T14:42:23.475084+0000","last_peered":"2024-09-13T14:42:23.475084+0000","last_clean":"2024-09-13T14:42:23.475084+0000","last_became_active":"2024-09-13T14:42:07.783466+0000","last_became_peered":"2024-09-13T14:42:07.783466+0000","last_unstale":"2024-09-13T14:42:23.475084+0000","last_undegraded":"2024-09-13T14:42:23.475084+0000","last_fullsized":"2024-09-13T14:42:23.475084+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1302,"log_dups_size":0,"ondisk_log_size":1302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:51.970990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1302,"num_write_kb":83328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":109,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342641+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:42:14.342641+0000","last_peered":"2024-09-13T14:42:14.342641+0000","last_clean":"2024-09-13T14:42:14.342641+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:42:14.342641+0000","last_undegraded":"2024-09-13T14:42:14.342641+0000","last_fullsized":"2024-09-13T14:42:14.342641+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":91,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.341834+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:42:14.341834+0000","last_peered":"2024-09-13T14:42:14.341834+0000","last_clean":"2024-09-13T14:42:14.341834+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:42:14.341834+0000","last_undegraded":"2024-09-13T14:42:14.341834+0000","last_fullsized":"2024-09-13T14:42:14.341834+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"143'1175","reported_seq":1194,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:22.604916+0000","last_change":"2024-09-13T14:42:07.780479+0000","last_active":"2024-09-13T14:42:22.604916+0000","last_peered":"2024-09-13T14:42:22.604916+0000","last_clean":"2024-09-13T14:42:22.604916+0000","last_became_active":"2024-09-13T14:42:07.780123+0000","last_became_peered":"2024-09-13T14:42:07.780123+0000","last_unstale":"2024-09-13T14:42:22.604916+0000","last_undegraded":"2024-09-13T14:42:22.604916+0000","last_fullsized":"2024-09-13T14:42:22.604916+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1175,"log_dups_size":0,"ondisk_log_size":1175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:20.035430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77004800,"num_objects":1175,"num_object_clones":0,"num_object_copies":3525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1175,"num_write_kb":75200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":110,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.343104+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:42:14.343104+0000","last_peered":"2024-09-13T14:42:14.343104+0000","last_clean":"2024-09-13T14:42:14.343104+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:42:14.343104+0000","last_undegraded":"2024-09-13T14:42:14.343104+0000","last_fullsized":"2024-09-13T14:42:14.343104+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169108+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:42:14.169108+0000","last_peered":"2024-09-13T14:42:14.169108+0000","last_clean":"2024-09-13T14:42:14.169108+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:42:14.169108+0000","last_undegraded":"2024-09-13T14:42:14.169108+0000","last_fullsized":"2024-09-13T14:42:14.169108+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"143'988","reported_seq":1000,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.476317+0000","last_change":"2024-09-13T14:42:07.781173+0000","last_active":"2024-09-13T14:42:21.476317+0000","last_peered":"2024-09-13T14:42:21.476317+0000","last_clean":"2024-09-13T14:42:21.476317+0000","last_became_active":"2024-09-13T14:42:07.780794+0000","last_became_peered":"2024-09-13T14:42:07.780794+0000","last_unstale":"2024-09-13T14:42:21.476317+0000","last_undegraded":"2024-09-13T14:42:21.476317+0000","last_fullsized":"2024-09-13T14:42:21.476317+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":988,"log_dups_size":0,"ondisk_log_size":988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:03.397389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64749568,"num_objects":988,"num_object_clones":0,"num_object_copies":2964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":988,"num_write_kb":63232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":94,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342876+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:42:14.342876+0000","last_peered":"2024-09-13T14:42:14.342876+0000","last_clean":"2024-09-13T14:42:14.342876+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:42:14.342876+0000","last_undegraded":"2024-09-13T14:42:14.342876+0000","last_fullsized":"2024-09-13T14:42:14.342876+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.789820+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:42:13.789820+0000","last_peered":"2024-09-13T14:42:13.789820+0000","last_clean":"2024-09-13T14:42:13.789820+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:42:13.789820+0000","last_undegraded":"2024-09-13T14:42:13.789820+0000","last_fullsized":"2024-09-13T14:42:13.789820+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"143'949","reported_seq":962,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.478259+0000","last_change":"2024-09-13T14:42:07.781817+0000","last_active":"2024-09-13T14:42:21.478259+0000","last_peered":"2024-09-13T14:42:21.478259+0000","last_clean":"2024-09-13T14:42:21.478259+0000","last_became_active":"2024-09-13T14:42:07.781534+0000","last_became_peered":"2024-09-13T14:42:07.781534+0000","last_unstale":"2024-09-13T14:42:21.478259+0000","last_undegraded":"2024-09-13T14:42:21.478259+0000","last_fullsized":"2024-09-13T14:42:21.478259+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":949,"log_dups_size":0,"ondisk_log_size":949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:27.372521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62193664,"num_objects":949,"num_object_clones":0,"num_object_copies":2847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":949,"num_write_kb":60736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342279+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:42:14.342279+0000","last_peered":"2024-09-13T14:42:14.342279+0000","last_clean":"2024-09-13T14:42:14.342279+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:42:14.342279+0000","last_undegraded":"2024-09-13T14:42:14.342279+0000","last_fullsized":"2024-09-13T14:42:14.342279+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873467+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:42:15.873467+0000","last_peered":"2024-09-13T14:42:15.873467+0000","last_clean":"2024-09-13T14:42:15.873467+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:42:15.873467+0000","last_undegraded":"2024-09-13T14:42:15.873467+0000","last_fullsized":"2024-09-13T14:42:15.873467+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"143'754","reported_seq":773,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:20.181726+0000","last_change":"2024-09-13T14:42:07.942573+0000","last_active":"2024-09-13T14:42:20.181726+0000","last_peered":"2024-09-13T14:42:20.181726+0000","last_clean":"2024-09-13T14:42:20.181726+0000","last_became_active":"2024-09-13T14:42:07.942229+0000","last_became_peered":"2024-09-13T14:42:07.942229+0000","last_unstale":"2024-09-13T14:42:20.181726+0000","last_undegraded":"2024-09-13T14:42:20.181726+0000","last_fullsized":"2024-09-13T14:42:20.181726+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":754,"log_dups_size":0,"ondisk_log_size":754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:32.927262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":754,"num_write_kb":48256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802325+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:42:15.802325+0000","last_peered":"2024-09-13T14:42:15.802325+0000","last_clean":"2024-09-13T14:42:15.802325+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:42:15.802325+0000","last_undegraded":"2024-09-13T14:42:15.802325+0000","last_fullsized":"2024-09-13T14:42:15.802325+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169366+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:42:14.169366+0000","last_peered":"2024-09-13T14:42:14.169366+0000","last_clean":"2024-09-13T14:42:14.169366+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:42:14.169366+0000","last_undegraded":"2024-09-13T14:42:14.169366+0000","last_fullsized":"2024-09-13T14:42:14.169366+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"143'1162","reported_seq":1181,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:22.615447+0000","last_change":"2024-09-13T14:42:07.942557+0000","last_active":"2024-09-13T14:42:22.615447+0000","last_peered":"2024-09-13T14:42:22.615447+0000","last_clean":"2024-09-13T14:42:22.615447+0000","last_became_active":"2024-09-13T14:42:07.942184+0000","last_became_peered":"2024-09-13T14:42:07.942184+0000","last_unstale":"2024-09-13T14:42:22.615447+0000","last_undegraded":"2024-09-13T14:42:22.615447+0000","last_fullsized":"2024-09-13T14:42:22.615447+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1162,"log_dups_size":0,"ondisk_log_size":1162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:25.265169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76152832,"num_objects":1162,"num_object_clones":0,"num_object_copies":3486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1162,"num_write_kb":74368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.789997+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:42:13.789997+0000","last_peered":"2024-09-13T14:42:13.789997+0000","last_clean":"2024-09-13T14:42:13.789997+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:42:13.789997+0000","last_undegraded":"2024-09-13T14:42:13.789997+0000","last_fullsized":"2024-09-13T14:42:13.789997+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169405+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:42:14.169405+0000","last_peered":"2024-09-13T14:42:14.169405+0000","last_clean":"2024-09-13T14:42:14.169405+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:42:14.169405+0000","last_undegraded":"2024-09-13T14:42:14.169405+0000","last_fullsized":"2024-09-13T14:42:14.169405+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"143'1028","reported_seq":1041,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.259378+0000","last_change":"2024-09-13T14:42:07.780892+0000","last_active":"2024-09-13T14:42:21.259378+0000","last_peered":"2024-09-13T14:42:21.259378+0000","last_clean":"2024-09-13T14:42:21.259378+0000","last_became_active":"2024-09-13T14:42:07.780540+0000","last_became_peered":"2024-09-13T14:42:07.780540+0000","last_unstale":"2024-09-13T14:42:21.259378+0000","last_undegraded":"2024-09-13T14:42:21.259378+0000","last_fullsized":"2024-09-13T14:42:21.259378+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1028,"log_dups_size":0,"ondisk_log_size":1028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:27.427471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67371008,"num_objects":1028,"num_object_clones":0,"num_object_copies":3084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1028,"num_write_kb":65792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":108,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169216+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:42:14.169216+0000","last_peered":"2024-09-13T14:42:14.169216+0000","last_clean":"2024-09-13T14:42:14.169216+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:42:14.169216+0000","last_undegraded":"2024-09-13T14:42:14.169216+0000","last_fullsized":"2024-09-13T14:42:14.169216+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":116,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169644+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:42:14.169644+0000","last_peered":"2024-09-13T14:42:14.169644+0000","last_clean":"2024-09-13T14:42:14.169644+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:42:14.169644+0000","last_undegraded":"2024-09-13T14:42:14.169644+0000","last_fullsized":"2024-09-13T14:42:14.169644+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"143'760","reported_seq":779,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:20.171664+0000","last_change":"2024-09-13T14:42:07.780605+0000","last_active":"2024-09-13T14:42:20.171664+0000","last_peered":"2024-09-13T14:42:20.171664+0000","last_clean":"2024-09-13T14:42:20.171664+0000","last_became_active":"2024-09-13T14:42:07.780309+0000","last_became_peered":"2024-09-13T14:42:07.780309+0000","last_unstale":"2024-09-13T14:42:20.171664+0000","last_undegraded":"2024-09-13T14:42:20.171664+0000","last_fullsized":"2024-09-13T14:42:20.171664+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":760,"log_dups_size":0,"ondisk_log_size":760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:29.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":760,"num_write_kb":48640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":106,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802509+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:42:15.802509+0000","last_peered":"2024-09-13T14:42:15.802509+0000","last_clean":"2024-09-13T14:42:15.802509+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:42:15.802509+0000","last_undegraded":"2024-09-13T14:42:15.802509+0000","last_fullsized":"2024-09-13T14:42:15.802509+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801156+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:42:15.801156+0000","last_peered":"2024-09-13T14:42:15.801156+0000","last_clean":"2024-09-13T14:42:15.801156+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:42:15.801156+0000","last_undegraded":"2024-09-13T14:42:15.801156+0000","last_fullsized":"2024-09-13T14:42:15.801156+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"143'731","reported_seq":750,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:20.178114+0000","last_change":"2024-09-13T14:42:07.782112+0000","last_active":"2024-09-13T14:42:20.178114+0000","last_peered":"2024-09-13T14:42:20.178114+0000","last_clean":"2024-09-13T14:42:20.178114+0000","last_became_active":"2024-09-13T14:42:07.781780+0000","last_became_peered":"2024-09-13T14:42:07.781780+0000","last_unstale":"2024-09-13T14:42:20.178114+0000","last_undegraded":"2024-09-13T14:42:20.178114+0000","last_fullsized":"2024-09-13T14:42:20.178114+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":731,"log_dups_size":0,"ondisk_log_size":731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:07.810835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47906816,"num_objects":731,"num_object_clones":0,"num_object_copies":2193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":731,"num_write_kb":46784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":440,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342527+0000","last_change":"2024-09-13T14:34:44.816346+0000","last_active":"2024-09-13T14:42:14.342527+0000","last_peered":"2024-09-13T14:42:14.342527+0000","last_clean":"2024-09-13T14:42:14.342527+0000","last_became_active":"2024-09-13T14:34:44.815951+0000","last_became_peered":"2024-09-13T14:34:44.815951+0000","last_unstale":"2024-09-13T14:42:14.342527+0000","last_undegraded":"2024-09-13T14:42:14.342527+0000","last_fullsized":"2024-09-13T14:42:14.342527+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:52:31.648689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342219+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:42:14.342219+0000","last_peered":"2024-09-13T14:42:14.342219+0000","last_clean":"2024-09-13T14:42:14.342219+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:42:14.342219+0000","last_undegraded":"2024-09-13T14:42:14.342219+0000","last_fullsized":"2024-09-13T14:42:14.342219+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802823+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:42:15.802823+0000","last_peered":"2024-09-13T14:42:15.802823+0000","last_clean":"2024-09-13T14:42:15.802823+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:42:15.802823+0000","last_undegraded":"2024-09-13T14:42:15.802823+0000","last_fullsized":"2024-09-13T14:42:15.802823+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.790000+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:42:13.790000+0000","last_peered":"2024-09-13T14:42:13.790000+0000","last_clean":"2024-09-13T14:42:13.790000+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:42:13.790000+0000","last_undegraded":"2024-09-13T14:42:13.790000+0000","last_fullsized":"2024-09-13T14:42:13.790000+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"143'1337","reported_seq":1356,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:23.470896+0000","last_change":"2024-09-13T14:42:07.942512+0000","last_active":"2024-09-13T14:42:23.470896+0000","last_peered":"2024-09-13T14:42:23.470896+0000","last_clean":"2024-09-13T14:42:23.470896+0000","last_became_active":"2024-09-13T14:42:07.942191+0000","last_became_peered":"2024-09-13T14:42:07.942191+0000","last_unstale":"2024-09-13T14:42:23.470896+0000","last_undegraded":"2024-09-13T14:42:23.470896+0000","last_fullsized":"2024-09-13T14:42:23.470896+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1337,"log_dups_size":0,"ondisk_log_size":1337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:07.538738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87621632,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1337,"num_write_kb":85568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":137,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342412+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:42:14.342412+0000","last_peered":"2024-09-13T14:42:14.342412+0000","last_clean":"2024-09-13T14:42:14.342412+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:42:14.342412+0000","last_undegraded":"2024-09-13T14:42:14.342412+0000","last_fullsized":"2024-09-13T14:42:14.342412+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":132,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873281+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:42:15.873281+0000","last_peered":"2024-09-13T14:42:15.873281+0000","last_clean":"2024-09-13T14:42:15.873281+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:42:15.873281+0000","last_undegraded":"2024-09-13T14:42:15.873281+0000","last_fullsized":"2024-09-13T14:42:15.873281+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":145,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873574+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:42:15.873574+0000","last_peered":"2024-09-13T14:42:15.873574+0000","last_clean":"2024-09-13T14:42:15.873574+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:42:15.873574+0000","last_undegraded":"2024-09-13T14:42:15.873574+0000","last_fullsized":"2024-09-13T14:42:15.873574+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"143'986","reported_seq":999,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.469746+0000","last_change":"2024-09-13T14:42:07.781099+0000","last_active":"2024-09-13T14:42:21.469746+0000","last_peered":"2024-09-13T14:42:21.469746+0000","last_clean":"2024-09-13T14:42:21.469746+0000","last_became_active":"2024-09-13T14:42:07.780786+0000","last_became_peered":"2024-09-13T14:42:07.780786+0000","last_unstale":"2024-09-13T14:42:21.469746+0000","last_undegraded":"2024-09-13T14:42:21.469746+0000","last_fullsized":"2024-09-13T14:42:21.469746+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":986,"log_dups_size":0,"ondisk_log_size":986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:24.082388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64618496,"num_objects":986,"num_object_clones":0,"num_object_copies":2958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":986,"num_write_kb":63104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":129,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800573+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:42:15.800573+0000","last_peered":"2024-09-13T14:42:15.800573+0000","last_clean":"2024-09-13T14:42:15.800573+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:42:15.800573+0000","last_undegraded":"2024-09-13T14:42:15.800573+0000","last_fullsized":"2024-09-13T14:42:15.800573+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800814+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:42:15.800814+0000","last_peered":"2024-09-13T14:42:15.800814+0000","last_clean":"2024-09-13T14:42:15.800814+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:42:15.800814+0000","last_undegraded":"2024-09-13T14:42:15.800814+0000","last_fullsized":"2024-09-13T14:42:15.800814+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801018+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:42:15.801018+0000","last_peered":"2024-09-13T14:42:15.801018+0000","last_clean":"2024-09-13T14:42:15.801018+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:42:15.801018+0000","last_undegraded":"2024-09-13T14:42:15.801018+0000","last_fullsized":"2024-09-13T14:42:15.801018+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"143'1394","reported_seq":1413,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:23.474992+0000","last_change":"2024-09-13T14:42:07.783147+0000","last_active":"2024-09-13T14:42:23.474992+0000","last_peered":"2024-09-13T14:42:23.474992+0000","last_clean":"2024-09-13T14:42:23.474992+0000","last_became_active":"2024-09-13T14:42:07.782831+0000","last_became_peered":"2024-09-13T14:42:07.782831+0000","last_unstale":"2024-09-13T14:42:23.474992+0000","last_undegraded":"2024-09-13T14:42:23.474992+0000","last_fullsized":"2024-09-13T14:42:23.474992+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":1394,"log_dups_size":0,"ondisk_log_size":1394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:03.638364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91357184,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1394,"num_write_kb":89216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":122,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342162+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:42:14.342162+0000","last_peered":"2024-09-13T14:42:14.342162+0000","last_clean":"2024-09-13T14:42:14.342162+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:42:14.342162+0000","last_undegraded":"2024-09-13T14:42:14.342162+0000","last_fullsized":"2024-09-13T14:42:14.342162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":28,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802179+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:42:15.802179+0000","last_peered":"2024-09-13T14:42:15.802179+0000","last_clean":"2024-09-13T14:42:15.802179+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:42:15.802179+0000","last_undegraded":"2024-09-13T14:42:15.802179+0000","last_fullsized":"2024-09-13T14:42:15.802179+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":116,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801567+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:42:15.801567+0000","last_peered":"2024-09-13T14:42:15.801567+0000","last_clean":"2024-09-13T14:42:15.801567+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:42:15.801567+0000","last_undegraded":"2024-09-13T14:42:15.801567+0000","last_fullsized":"2024-09-13T14:42:15.801567+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"143'981","reported_seq":994,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.469782+0000","last_change":"2024-09-13T14:42:07.781837+0000","last_active":"2024-09-13T14:42:21.469782+0000","last_peered":"2024-09-13T14:42:21.469782+0000","last_clean":"2024-09-13T14:42:21.469782+0000","last_became_active":"2024-09-13T14:42:07.781584+0000","last_became_peered":"2024-09-13T14:42:07.781584+0000","last_unstale":"2024-09-13T14:42:21.469782+0000","last_undegraded":"2024-09-13T14:42:21.469782+0000","last_fullsized":"2024-09-13T14:42:21.469782+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":981,"log_dups_size":0,"ondisk_log_size":981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:21.323962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64290816,"num_objects":981,"num_object_clones":0,"num_object_copies":2943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":981,"num_write_kb":62784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":123,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342579+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:42:14.342579+0000","last_peered":"2024-09-13T14:42:14.342579+0000","last_clean":"2024-09-13T14:42:14.342579+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:42:14.342579+0000","last_undegraded":"2024-09-13T14:42:14.342579+0000","last_fullsized":"2024-09-13T14:42:14.342579+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":154,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801778+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:42:15.801778+0000","last_peered":"2024-09-13T14:42:15.801778+0000","last_clean":"2024-09-13T14:42:15.801778+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:42:15.801778+0000","last_undegraded":"2024-09-13T14:42:15.801778+0000","last_fullsized":"2024-09-13T14:42:15.801778+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":24,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800913+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:42:15.800913+0000","last_peered":"2024-09-13T14:42:15.800913+0000","last_clean":"2024-09-13T14:42:15.800913+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:42:15.800913+0000","last_undegraded":"2024-09-13T14:42:15.800913+0000","last_fullsized":"2024-09-13T14:42:15.800913+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"143'934","reported_seq":947,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.267359+0000","last_change":"2024-09-13T14:42:07.781001+0000","last_active":"2024-09-13T14:42:21.267359+0000","last_peered":"2024-09-13T14:42:21.267359+0000","last_clean":"2024-09-13T14:42:21.267359+0000","last_became_active":"2024-09-13T14:42:07.780697+0000","last_became_peered":"2024-09-13T14:42:07.780697+0000","last_unstale":"2024-09-13T14:42:21.267359+0000","last_undegraded":"2024-09-13T14:42:21.267359+0000","last_fullsized":"2024-09-13T14:42:21.267359+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":934,"log_dups_size":0,"ondisk_log_size":934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:29:01.574906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":934,"num_write_kb":59776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801645+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:42:15.801645+0000","last_peered":"2024-09-13T14:42:15.801645+0000","last_clean":"2024-09-13T14:42:15.801645+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:42:15.801645+0000","last_undegraded":"2024-09-13T14:42:15.801645+0000","last_fullsized":"2024-09-13T14:42:15.801645+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.343138+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:42:14.343138+0000","last_peered":"2024-09-13T14:42:14.343138+0000","last_clean":"2024-09-13T14:42:14.343138+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:42:14.343138+0000","last_undegraded":"2024-09-13T14:42:14.343138+0000","last_fullsized":"2024-09-13T14:42:14.343138+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":123,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873489+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:42:15.873489+0000","last_peered":"2024-09-13T14:42:15.873489+0000","last_clean":"2024-09-13T14:42:15.873489+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:42:15.873489+0000","last_undegraded":"2024-09-13T14:42:15.873489+0000","last_fullsized":"2024-09-13T14:42:15.873489+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"143'983","reported_seq":996,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:21.272170+0000","last_change":"2024-09-13T14:42:07.782820+0000","last_active":"2024-09-13T14:42:21.272170+0000","last_peered":"2024-09-13T14:42:21.272170+0000","last_clean":"2024-09-13T14:42:21.272170+0000","last_became_active":"2024-09-13T14:42:07.782375+0000","last_became_peered":"2024-09-13T14:42:07.782375+0000","last_unstale":"2024-09-13T14:42:21.272170+0000","last_undegraded":"2024-09-13T14:42:21.272170+0000","last_fullsized":"2024-09-13T14:42:21.272170+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":983,"log_dups_size":0,"ondisk_log_size":983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:39.823277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":983,"num_write_kb":62912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":92,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801924+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:42:15.801924+0000","last_peered":"2024-09-13T14:42:15.801924+0000","last_clean":"2024-09-13T14:42:15.801924+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:42:15.801924+0000","last_undegraded":"2024-09-13T14:42:15.801924+0000","last_fullsized":"2024-09-13T14:42:15.801924+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801490+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:42:15.801490+0000","last_peered":"2024-09-13T14:42:15.801490+0000","last_clean":"2024-09-13T14:42:15.801490+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:42:15.801490+0000","last_undegraded":"2024-09-13T14:42:15.801490+0000","last_fullsized":"2024-09-13T14:42:15.801490+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802244+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:42:15.802244+0000","last_peered":"2024-09-13T14:42:15.802244+0000","last_clean":"2024-09-13T14:42:15.802244+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:42:15.802244+0000","last_undegraded":"2024-09-13T14:42:15.802244+0000","last_fullsized":"2024-09-13T14:42:15.802244+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"143'749","reported_seq":768,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:20.178841+0000","last_change":"2024-09-13T14:42:07.782967+0000","last_active":"2024-09-13T14:42:20.178841+0000","last_peered":"2024-09-13T14:42:20.178841+0000","last_clean":"2024-09-13T14:42:20.178841+0000","last_became_active":"2024-09-13T14:42:07.782365+0000","last_became_peered":"2024-09-13T14:42:07.782365+0000","last_unstale":"2024-09-13T14:42:20.178841+0000","last_undegraded":"2024-09-13T14:42:20.178841+0000","last_fullsized":"2024-09-13T14:42:20.178841+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":749,"log_dups_size":0,"ondisk_log_size":749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:02.386475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49086464,"num_objects":749,"num_object_clones":0,"num_object_copies":2247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":749,"num_write_kb":47936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":137,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873791+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:42:15.873791+0000","last_peered":"2024-09-13T14:42:15.873791+0000","last_clean":"2024-09-13T14:42:15.873791+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:42:15.873791+0000","last_undegraded":"2024-09-13T14:42:15.873791+0000","last_fullsized":"2024-09-13T14:42:15.873791+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.790162+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:42:13.790162+0000","last_peered":"2024-09-13T14:42:13.790162+0000","last_clean":"2024-09-13T14:42:13.790162+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:42:13.790162+0000","last_undegraded":"2024-09-13T14:42:13.790162+0000","last_fullsized":"2024-09-13T14:42:13.790162+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":93,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169103+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:42:14.169103+0000","last_peered":"2024-09-13T14:42:14.169103+0000","last_clean":"2024-09-13T14:42:14.169103+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:42:14.169103+0000","last_undegraded":"2024-09-13T14:42:14.169103+0000","last_fullsized":"2024-09-13T14:42:14.169103+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169269+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:42:14.169269+0000","last_peered":"2024-09-13T14:42:14.169269+0000","last_clean":"2024-09-13T14:42:14.169269+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:42:14.169269+0000","last_undegraded":"2024-09-13T14:42:14.169269+0000","last_fullsized":"2024-09-13T14:42:14.169269+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":116,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.168962+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:42:14.168962+0000","last_peered":"2024-09-13T14:42:14.168962+0000","last_clean":"2024-09-13T14:42:14.168962+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:42:14.168962+0000","last_undegraded":"2024-09-13T14:42:14.168962+0000","last_fullsized":"2024-09-13T14:42:14.168962+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":24,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801128+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:42:15.801128+0000","last_peered":"2024-09-13T14:42:15.801128+0000","last_clean":"2024-09-13T14:42:15.801128+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:42:15.801128+0000","last_undegraded":"2024-09-13T14:42:15.801128+0000","last_fullsized":"2024-09-13T14:42:15.801128+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":49,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802304+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:42:15.802304+0000","last_peered":"2024-09-13T14:42:15.802304+0000","last_clean":"2024-09-13T14:42:15.802304+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:42:15.802304+0000","last_undegraded":"2024-09-13T14:42:15.802304+0000","last_fullsized":"2024-09-13T14:42:15.802304+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342692+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:42:14.342692+0000","last_peered":"2024-09-13T14:42:14.342692+0000","last_clean":"2024-09-13T14:42:14.342692+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:42:14.342692+0000","last_undegraded":"2024-09-13T14:42:14.342692+0000","last_fullsized":"2024-09-13T14:42:14.342692+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800788+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:42:15.800788+0000","last_peered":"2024-09-13T14:42:15.800788+0000","last_clean":"2024-09-13T14:42:15.800788+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:42:15.800788+0000","last_undegraded":"2024-09-13T14:42:15.800788+0000","last_fullsized":"2024-09-13T14:42:15.800788+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":132,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873778+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:42:15.873778+0000","last_peered":"2024-09-13T14:42:15.873778+0000","last_clean":"2024-09-13T14:42:15.873778+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:42:15.873778+0000","last_undegraded":"2024-09-13T14:42:15.873778+0000","last_fullsized":"2024-09-13T14:42:15.873778+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800356+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:42:15.800356+0000","last_peered":"2024-09-13T14:42:15.800356+0000","last_clean":"2024-09-13T14:42:15.800356+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:42:15.800356+0000","last_undegraded":"2024-09-13T14:42:15.800356+0000","last_fullsized":"2024-09-13T14:42:15.800356+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":108,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:13.790168+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:42:13.790168+0000","last_peered":"2024-09-13T14:42:13.790168+0000","last_clean":"2024-09-13T14:42:13.790168+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:42:13.790168+0000","last_undegraded":"2024-09-13T14:42:13.790168+0000","last_fullsized":"2024-09-13T14:42:13.790168+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":120,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802419+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:42:15.802419+0000","last_peered":"2024-09-13T14:42:15.802419+0000","last_clean":"2024-09-13T14:42:15.802419+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:42:15.802419+0000","last_undegraded":"2024-09-13T14:42:15.802419+0000","last_fullsized":"2024-09-13T14:42:15.802419+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":134,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873694+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:42:15.873694+0000","last_peered":"2024-09-13T14:42:15.873694+0000","last_clean":"2024-09-13T14:42:15.873694+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:42:15.873694+0000","last_undegraded":"2024-09-13T14:42:15.873694+0000","last_fullsized":"2024-09-13T14:42:15.873694+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":102,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.800993+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:42:15.800993+0000","last_peered":"2024-09-13T14:42:15.800993+0000","last_clean":"2024-09-13T14:42:15.800993+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:42:15.800993+0000","last_undegraded":"2024-09-13T14:42:15.800993+0000","last_fullsized":"2024-09-13T14:42:15.800993+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":104,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801172+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:42:15.801172+0000","last_peered":"2024-09-13T14:42:15.801172+0000","last_clean":"2024-09-13T14:42:15.801172+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:42:15.801172+0000","last_undegraded":"2024-09-13T14:42:15.801172+0000","last_fullsized":"2024-09-13T14:42:15.801172+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":28,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802771+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:42:15.802771+0000","last_peered":"2024-09-13T14:42:15.802771+0000","last_clean":"2024-09-13T14:42:15.802771+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:42:15.802771+0000","last_undegraded":"2024-09-13T14:42:15.802771+0000","last_fullsized":"2024-09-13T14:42:15.802771+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":129,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.802138+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:42:15.802138+0000","last_peered":"2024-09-13T14:42:15.802138+0000","last_clean":"2024-09-13T14:42:15.802138+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:42:15.802138+0000","last_undegraded":"2024-09-13T14:42:15.802138+0000","last_fullsized":"2024-09-13T14:42:15.802138+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":138,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873610+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:42:15.873610+0000","last_peered":"2024-09-13T14:42:15.873610+0000","last_clean":"2024-09-13T14:42:15.873610+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:42:15.873610+0000","last_undegraded":"2024-09-13T14:42:15.873610+0000","last_fullsized":"2024-09-13T14:42:15.873610+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169158+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:42:14.169158+0000","last_peered":"2024-09-13T14:42:14.169158+0000","last_clean":"2024-09-13T14:42:14.169158+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:42:14.169158+0000","last_undegraded":"2024-09-13T14:42:14.169158+0000","last_fullsized":"2024-09-13T14:42:14.169158+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":108,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.342196+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:42:14.342196+0000","last_peered":"2024-09-13T14:42:14.342196+0000","last_clean":"2024-09-13T14:42:14.342196+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:42:14.342196+0000","last_undegraded":"2024-09-13T14:42:14.342196+0000","last_fullsized":"2024-09-13T14:42:14.342196+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":121,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.169106+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:42:14.169106+0000","last_peered":"2024-09-13T14:42:14.169106+0000","last_clean":"2024-09-13T14:42:14.169106+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:42:14.169106+0000","last_undegraded":"2024-09-13T14:42:14.169106+0000","last_fullsized":"2024-09-13T14:42:14.169106+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":93,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.341869+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:42:14.341869+0000","last_peered":"2024-09-13T14:42:14.341869+0000","last_clean":"2024-09-13T14:42:14.341869+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:42:14.341869+0000","last_undegraded":"2024-09-13T14:42:14.341869+0000","last_fullsized":"2024-09-13T14:42:14.341869+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801075+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:42:15.801075+0000","last_peered":"2024-09-13T14:42:15.801075+0000","last_clean":"2024-09-13T14:42:15.801075+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:42:15.801075+0000","last_undegraded":"2024-09-13T14:42:15.801075+0000","last_fullsized":"2024-09-13T14:42:15.801075+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":125,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.801498+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:42:15.801498+0000","last_peered":"2024-09-13T14:42:15.801498+0000","last_clean":"2024-09-13T14:42:15.801498+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:42:15.801498+0000","last_undegraded":"2024-09-13T14:42:15.801498+0000","last_fullsized":"2024-09-13T14:42:15.801498+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":142,"state":"active+clean","last_fresh":"2024-09-13T14:42:14.168999+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:42:14.168999+0000","last_peered":"2024-09-13T14:42:14.168999+0000","last_clean":"2024-09-13T14:42:14.168999+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:42:14.168999+0000","last_undegraded":"2024-09-13T14:42:14.168999+0000","last_fullsized":"2024-09-13T14:42:14.168999+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":138,"reported_epoch":143,"state":"active+clean","last_fresh":"2024-09-13T14:42:15.873604+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:42:15.873604+0000","last_peered":"2024-09-13T14:42:15.873604+0000","last_clean":"2024-09-13T14:42:15.873604+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:42:15.873604+0000","last_undegraded":"2024-09-13T14:42:15.873604+0000","last_fullsized":"2024-09-13T14:42:15.873604+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1062535168,"num_objects":16213,"num_object_clones":0,"num_object_copies":48639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16213,"num_write_kb":1037632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3121414144,"data_stored":3121414144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16213,"ondisk_log_size":16213,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234267,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":970728,"kb_used_data":484320,"kb_used_omap":1,"kb_used_meta":486398,"kb_avail":92774424,"statfs":{"total":95995035648,"available":95001010176,"internally_reserved":0,"allocated":495943680,"data_stored":495202562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":498072011},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495905,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":581604,"kb_used_data":167388,"kb_used_omap":1,"kb_used_meta":414206,"kb_avail":93163548,"statfs":{"total":95995035648,"available":95399473152,"internally_reserved":0,"allocated":171405312,"data_stored":170668253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":424147399},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757543,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1157140,"kb_used_data":536204,"kb_used_omap":1,"kb_used_meta":620926,"kb_avail":92588012,"statfs":{"total":95995035648,"available":94810124288,"internally_reserved":0,"allocated":549072896,"data_stored":548347172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":635828678},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019179,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747488,"kb_used_data":166744,"kb_used_omap":1,"kb_used_meta":580734,"kb_avail":92997664,"statfs":{"total":95995035648,"available":95229607936,"internally_reserved":0,"allocated":170745856,"data_stored":170012372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":594672070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650391,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404368,"kb_used_data":306312,"kb_used_omap":4,"kb_used_meta":98043,"kb_avail":93340784,"statfs":{"total":95995035648,"available":95580962816,"internally_reserved":0,"allocated":313663488,"data_stored":312941622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":100396591},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879326,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":210392,"kb_used_data":126800,"kb_used_omap":6,"kb_used_meta":83577,"kb_avail":93534760,"statfs":{"total":95995035648,"available":95779594240,"internally_reserved":0,"allocated":129843200,"data_stored":129113758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":85583417},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435343,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":884556,"kb_used_data":671940,"kb_used_omap":5,"kb_used_meta":212602,"kb_avail":92860596,"statfs":{"total":95995035648,"available":95089250304,"internally_reserved":0,"allocated":688066560,"data_stored":687348207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":217704801},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925918,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":501272,"kb_used_data":303120,"kb_used_omap":5,"kb_used_meta":198138,"kb_avail":93243880,"statfs":{"total":95995035648,"available":95481733120,"internally_reserved":0,"allocated":310394880,"data_stored":309665880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":202893482},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":350158848,"data_stored":350158848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":702939136,"data_stored":702939136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":152698880,"data_stored":152698880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":392822784,"data_stored":392822784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":185925632,"data_stored":185925632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":583270400,"data_stored":583270400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":187105280,"data_stored":187105280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":566493184,"data_stored":566493184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:42:31.742 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:42:31.743 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-13T14:42:31.762 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:31.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:31.864 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:31.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:31.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:31.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:31.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:31 smithi042 ceph-mon[108570]: pgmap v486: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:42:32.000 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:31 smithi042 ceph-mon[113926]: pgmap v486: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:42:32.021 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:42:32.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:32.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:32.147 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:32.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:32.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:32.226 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:31 smithi159 ceph-mon[102519]: pgmap v486: 89 pgs: 89 active+clean; 1.8 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-09-13T14:42:32.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.377 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:32.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:32.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:32.498 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:42:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=sqlstore.transactions t=2024-09-13T14:42:32.225369762Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T14:42:32.506 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:32.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 41699 41683 162.796 109.125 0.00891688 0.00613723 2024-09-13T14:42:32.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:32.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:32.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:32.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:32.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:32.921 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:32.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:32.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:33.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:33.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:33.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:33.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:33.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.287 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:33.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:33.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:33.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:33.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:33.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:33.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 15 43516 43501 159.903 113.625 0.0157651 0.00624875 2024-09-13T14:42:33.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:33.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:33.702 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:33.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:33.756 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:33.756 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:33.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:33.880 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:42:33.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:33.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:33.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:34.025 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:34.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[113926]: pgmap v487: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 183 MiB/s wr, 2.92k op/s 2024-09-13T14:42:34.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/835673218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.116 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[108570]: pgmap v487: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 183 MiB/s wr, 2.92k op/s 2024-09-13T14:42:34.116 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/835673218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.116 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.126 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:34.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:33 smithi159 ceph-mon[102519]: pgmap v487: 89 pgs: 89 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 183 MiB/s wr, 2.92k op/s 2024-09-13T14:42:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/835673218' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:42:34.261 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:34.311 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:34.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:34.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:34.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:34.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 45417 45401 157.616 118.75 0.0036948 0.00633923 2024-09-13T14:42:34.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:34.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:34.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:34.832 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:34.863 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:34.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:34.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:34.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:34.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:35.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:35.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:35.198 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:35.199 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:35.199 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[113926]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:42:35.199 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:35.199 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:35.199 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:34 smithi042 ceph-mon[108570]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:42:35.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:35.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:34 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:42:35.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:34 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:42:35.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:34 smithi159 ceph-mon[102519]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T14:42:35.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:35.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:35.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:35.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:35.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:35.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 47373 47357 155.754 122.25 0.0069373 0.00641482 2024-09-13T14:42:35.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:35.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:35.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:35.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:35.909 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:35 smithi042 ceph-mon[113926]: pgmap v489: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-13T14:42:35.909 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:35.909 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:35 smithi042 ceph-mon[108570]: pgmap v489: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-13T14:42:35.909 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:35.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:35.969 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:35.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:36.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:36.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:36.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:36.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:36.248 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:35 smithi159 ceph-mon[102519]: pgmap v489: 89 pgs: 89 active+clean; 2.4 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-09-13T14:42:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:36.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:36.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:36.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:36.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:36.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:36.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:36.570 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:42:36.570873+0000 min lat: 0.00132914 max lat: 0.0804403 avg lat: 0.00649816 2024-09-13T14:42:36.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:42:36.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 49229 49213 153.766 116 0.00678443 0.00649816 2024-09-13T14:42:36.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:36.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:36.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:36.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:36.824 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:36.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:36.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:36.975 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:36.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:37.061 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:42:37.061 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:37.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:37.098 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:42:37.098 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:37.114 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:37.134 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:42:37.135 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:37.175 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:42:37.176 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:37.209 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:42:37.210 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:37.216 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:37.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:37.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:37.248 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:42:37.248 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:37.284 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:42:37.285 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:37.324 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:42:37.324 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:37.347 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:37.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:37.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:37.400 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:42:37.400 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:42:37.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:37.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:37.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 50895 50879 151.402 104.125 0.0115108 0.0065999 2024-09-13T14:42:37.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:37.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:37.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:37.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:38.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:37 smithi159 ceph-mon[102519]: pgmap v490: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:42:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3073789490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:38.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:38.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:38.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:37 smithi042 ceph-mon[113926]: pgmap v490: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:42:38.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3073789490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:38.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:37 smithi042 ceph-mon[108570]: pgmap v490: 89 pgs: 89 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 142 MiB/s wr, 2.27k op/s 2024-09-13T14:42:38.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3073789490' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:38.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:42:38.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:38.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:38.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:38.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 52741 52725 149.764 115.375 0.0102173 0.00667204 2024-09-13T14:42:38.574 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:38.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:38.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:38.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:38.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:38.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:38.976 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:38.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:39.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:39.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:39.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:39.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:39.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:39.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:39.396 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:39.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:39.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:39.538 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:39.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:39.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 54476 54460 147.967 108.438 0.00787531 0.00675323 2024-09-13T14:42:39.640 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:39.672 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:39.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:39.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:39.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:39.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:39.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:39.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:40.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:40.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:39 smithi042 ceph-mon[108570]: pgmap v491: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:42:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:39 smithi042 ceph-mon[113926]: pgmap v491: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:42:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:39] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-09-13T14:42:40.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:40.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:40.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:39 smithi159 ceph-mon[102519]: pgmap v491: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-09-13T14:42:40.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:40.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:40.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:40.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:42:40.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:40.494 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:40.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:40.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 56104 56088 146.041 101.75 0.00650994 0.00684204 2024-09-13T14:42:40.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:42:40.596 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:40.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:40.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:40.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:40.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:40.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:40.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:40.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:41.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.129 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.136 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:41.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:41.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:41.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:41.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:41.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:41.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:41.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 57531 57515 143.766 89.1875 0.00747618 0.00694982 2024-09-13T14:42:41.672 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:41.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:41.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:41.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:41.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:41.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:41.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.001 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:41 smithi042 ceph-mon[113926]: pgmap v492: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:42:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:41 smithi042 ceph-mon[108570]: pgmap v492: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:42:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:42.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:42.142 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:42.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:41 smithi159 ceph-mon[102519]: pgmap v492: 89 pgs: 89 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-13T14:42:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:42:42.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:42.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:42.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:42.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:42.551 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:42.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 59023 59007 141.823 93.25 0.00302119 0.00704341 2024-09-13T14:42:42.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:42.687 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:42.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:42.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:42.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:42.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:42.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:42.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:43.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:43.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:43.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:43.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:43.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:43.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:43.404 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:43.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:43.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:43.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 60562 60546 140.133 96.1875 0.00579017 0.00713128 2024-09-13T14:42:43.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:43.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:43.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:43.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.819 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:43.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:43.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:43.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:43.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:44.098 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:44.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:44.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:44.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:44.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:43 smithi159 ceph-mon[102519]: pgmap v493: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-13T14:42:44.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:44.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:43 smithi042 ceph-mon[113926]: pgmap v493: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-13T14:42:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:43 smithi042 ceph-mon[108570]: pgmap v493: 89 pgs: 89 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-09-13T14:42:44.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:44.478 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:44.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:44.489 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:44.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:44.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 62025 62009 138.393 91.4375 0.0112523 0.00722073 2024-09-13T14:42:44.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.656 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:44.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:44.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:44.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:44.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:44.908 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 1 pg_num to 11 2024-09-13T14:42:44.935 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:44.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:45.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:45.043 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1138933819' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.043 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:45.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:45.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.211 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:45.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:45.247 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1138933819' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.247 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.247 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1138933819' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.248 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-13T14:42:45.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:45.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:45.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:45.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:45.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:45.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:45.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 63417 63401 136.621 87 0.0119314 0.0073136 2024-09-13T14:42:45.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:45.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:45.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:45.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:45.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:45.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:45.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:45.928 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:45.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:45.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:46.030 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:46.054 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:46.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:46.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:46.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:46.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:46.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[113926]: pgmap v494: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:46.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:42:46.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[113926]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:42:46.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[108570]: pgmap v494: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:46.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:42:46.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:45 smithi042 ceph-mon[108570]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:42:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:45 smithi159 ceph-mon[102519]: pgmap v494: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:45 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-13T14:42:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:45 smithi159 ceph-mon[102519]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T14:42:46.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.316 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:46.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:46.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:46.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:46.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:46.571 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 64396 64380 134.106 61.1875 0.00469891 0.00745145 2024-09-13T14:42:46.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:46.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:46.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:46.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:46.873 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:46.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:46.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:46.931 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:42:46.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:47.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:47.110 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:47.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:46 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:46 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2928833632' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:46 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2928833632' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2928833632' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.254 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:46 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:42:47.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:47.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:47.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:47.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:47.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:47.572 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 65717 65701 132.444 82.5625 0.00621331 0.00754428 2024-09-13T14:42:47.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:47.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:47.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:47.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:47.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:47.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:47.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:47.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:48.000 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:48.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:48.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:48.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:48.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: pgmap v496: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:42:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:42:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:47 smithi159 ceph-mon[102519]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:42:48.286 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: pgmap v496: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[113926]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: pgmap v496: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:42:48.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:47 smithi042 ceph-mon[108570]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T14:42:48.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.525 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:48.572 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 67103 67087 131.012 86.625 0.0132075 0.00762711 2024-09-13T14:42:48.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:48.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:48.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:48.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:48.862 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:48.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:48.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:49.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:49.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:49.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.148 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:49.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:48 smithi159 ceph-mon[102519]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:42:49.249 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:49.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:48 smithi042 ceph-mon[108570]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:42:49.265 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:48 smithi042 ceph-mon[113926]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T14:42:49.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:49.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:49.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:49.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:49.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:49.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 68561 68545 129.8 91.125 0.0154094 0.00769879 2024-09-13T14:42:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:49.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:49.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:49.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:49.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:49.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:49.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:49.999 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:50.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:50.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[113926]: pgmap v499: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:42:50.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.015 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:49] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-09-13T14:42:50.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[108570]: pgmap v499: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:42:50.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:50.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:50.236 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:49 smithi159 ceph-mon[102519]: pgmap v499: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T14:42:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:50.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:50.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:50.414 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:50.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:50.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 70012 69996 128.649 90.6875 0.00726407 0.00776803 2024-09-13T14:42:50.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:50.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.701 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:50.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:50.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:50.836 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:50.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:50.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:50.975 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:50.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:51.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:51.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:51.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:51.267 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:51.369 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:51.409 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:51.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:51.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:51.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 15 71472 71457 127.582 91.3125 0.00638019 0.00783311 2024-09-13T14:42:51.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:51.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:51.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.851 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:51.865 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:42:51.865 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:42:51.865 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:42:51.866 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:42:51.867 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:42:51.867 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: pgmap v500: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: 1.0 scrub starts 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: 1.0 scrub ok 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:51.867 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/642534204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: pgmap v500: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: 1.0 scrub starts 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: 1.0 scrub ok 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:51.868 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/642534204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:51.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:51.989 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:51.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:52.091 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:52.128 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:52.230 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: pgmap v500: 99 pgs: 10 unknown, 89 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: 1.0 scrub starts 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: 1.0 scrub ok 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:42:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/642534204' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:42:52.265 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:52.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:52.294 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:42:52.294 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:42:52.294 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-09-13T14:42:52.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:42:52.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:52.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:52.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:42:52.528 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:52.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:52.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:52.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 72842 72826 126.414 85.5625 0.0124797 0.00790566 2024-09-13T14:42:52.659 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:52.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:52.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:52.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:52.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:52.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:53.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[108570]: 1.9 scrub starts 2024-09-13T14:42:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[108570]: 1.9 scrub ok 2024-09-13T14:42:53.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[113926]: 1.9 scrub starts 2024-09-13T14:42:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[113926]: 1.9 scrub ok 2024-09-13T14:42:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:52 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:53.136 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.143 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:42:53.144 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:42:53.144 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:42:53.144 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:42:53.144 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:42:53.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:52 smithi159 ceph-mon[102519]: 1.9 scrub starts 2024-09-13T14:42:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:52 smithi159 ceph-mon[102519]: 1.9 scrub ok 2024-09-13T14:42:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:52 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-13T14:42:53.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:53.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:53.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:53.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:53.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:53.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.561 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 74203 74187 125.296 85.0625 0.00444115 0.00797608 2024-09-13T14:42:53.663 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:53.701 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:53.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:53.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:53.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:53.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:53.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:42:53.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:54.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:54.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[113926]: pgmap v501: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:42:54.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:42:54.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[113926]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:42:54.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[113926]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:42:54.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[108570]: pgmap v501: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:42:54.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:42:54.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[108570]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:42:54.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:53 smithi042 ceph-mon[108570]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:42:54.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:54.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:53 smithi159 ceph-mon[102519]: pgmap v501: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s 2024-09-13T14:42:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-13T14:42:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:53 smithi159 ceph-mon[102519]: osdmap e148: 8 total, 8 up, 8 in 2024-09-13T14:42:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:53 smithi159 ceph-mon[102519]: osdmap e149: 8 total, 8 up, 8 in 2024-09-13T14:42:54.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:54.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:54.430 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:54.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:54.540 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:54.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:54.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 75529 75513 124.18 82.875 0.011146 0.00804793 2024-09-13T14:42:54.580 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:42:54.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:54.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:54.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:54.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:54.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:54.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:54.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:55.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:55.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:55.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:55.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:55.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:55.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:55.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:55.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:55.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:55.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 76794 76778 123.023 79.0625 0.00861117 0.00812328 2024-09-13T14:42:55.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:55.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:55.720 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:55.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:55.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:55.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:55.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:55.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:55.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:55 smithi159 ceph-mon[102519]: pgmap v504: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:42:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:55 smithi159 ceph-mon[102519]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:42:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:55 smithi159 ceph-mon[102519]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:42:56.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:56.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[113926]: pgmap v504: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:42:56.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[113926]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:42:56.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[113926]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:42:56.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[108570]: pgmap v504: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:42:56.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[108570]: osdmap e150: 8 total, 8 up, 8 in 2024-09-13T14:42:56.097 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:55 smithi042 ceph-mon[108570]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T14:42:56.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:56.174 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:56.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:56.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:42:56.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:56.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:42:56.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:42:56.557 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:56.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:42:56.573813+0000 min lat: 0.00132914 max lat: 0.273076 avg lat: 0.00819769 2024-09-13T14:42:56.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:42:56.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 78048 78032 121.906 78.375 0.00585261 0.00819769 2024-09-13T14:42:56.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:56.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:56.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:56.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:56.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:56.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:56.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:56.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:56.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:56.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:57.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:57.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:57.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:57.152 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:57.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:57.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:57.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.287 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:57.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:57.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:57.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:57.451 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: pgmap v507: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.1 deep-scrub starts 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.1 deep-scrub ok 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.4 scrub starts 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.4 scrub ok 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.3 scrub starts 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.3 scrub ok 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[113926]: 1.2 scrub ok 2024-09-13T14:42:57.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: pgmap v507: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.1 deep-scrub starts 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.1 deep-scrub ok 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.4 scrub starts 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.4 scrub ok 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.3 scrub starts 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.3 scrub ok 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:42:57.453 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:57 smithi042 ceph-mon[108570]: 1.2 scrub ok 2024-09-13T14:42:57.527 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:57.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:57.573 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 79408 79392 121.006 85 0.0113668 0.0082597 2024-09-13T14:42:57.667 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.707 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:57.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: pgmap v507: 99 pgs: 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.1 deep-scrub starts 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.1 deep-scrub ok 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.4 scrub starts 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.4 scrub ok 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.3 scrub starts 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.3 scrub ok 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:42:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:57 smithi159 ceph-mon[102519]: 1.2 scrub ok 2024-09-13T14:42:57.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:57.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:57.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:57.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:58.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:58.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:58.108 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:58.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:58.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:58.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:58.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:58.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:58 smithi042 ceph-mon[108570]: 1.a scrub starts 2024-09-13T14:42:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:58 smithi042 ceph-mon[108570]: 1.a scrub ok 2024-09-13T14:42:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:58 smithi042 ceph-mon[113926]: 1.a scrub starts 2024-09-13T14:42:58.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:58 smithi042 ceph-mon[113926]: 1.a scrub ok 2024-09-13T14:42:58.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:58.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:58.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:58.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:58.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:58.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 80694 80678 120.038 80.375 0.0093968 0.00832558 2024-09-13T14:42:58.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:42:58.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:42:58.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:58 smithi159 ceph-mon[102519]: 1.a scrub starts 2024-09-13T14:42:58.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:58 smithi159 ceph-mon[102519]: 1.a scrub ok 2024-09-13T14:42:58.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:42:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:58.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:42:58.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:42:58.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:58.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:58.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:42:59.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:59.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:59.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:42:59.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:42:59.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:42:59.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:59.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:42:59.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:42:59.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 81970 81954 119.101 79.75 0.0124002 0.00839167 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[113926]: 1.6 scrub starts 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[113926]: 1.6 scrub ok 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[113926]: pgmap v509: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[108570]: 1.6 scrub starts 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[108570]: 1.6 scrub ok 2024-09-13T14:42:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-mon[108570]: pgmap v509: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T14:42:59.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:59.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:59 smithi159 ceph-mon[102519]: 1.6 scrub starts 2024-09-13T14:42:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:59 smithi159 ceph-mon[102519]: 1.6 scrub ok 2024-09-13T14:42:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:42:59 smithi159 ceph-mon[102519]: pgmap v509: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T14:42:59.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:42:59.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:42:59.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:42:59.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:42:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:42:59.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:42:59.999 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:42:59.999 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:42:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:42:59] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.51.0" 2024-09-13T14:43:00.005 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:43:00.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:00.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:00.186 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:00.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:00.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:00.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:00.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:00.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:00.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:00.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:00.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 83269 83253 118.24 81.1875 0.0199238 0.0084524 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[113926]: 1.7 deep-scrub starts 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[113926]: 1.7 deep-scrub ok 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[113926]: 1.8 scrub starts 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[113926]: 1.8 scrub ok 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[113926]: from='client.54589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[108570]: 1.7 deep-scrub starts 2024-09-13T14:43:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[108570]: 1.7 deep-scrub ok 2024-09-13T14:43:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[108570]: 1.8 scrub starts 2024-09-13T14:43:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[108570]: 1.8 scrub ok 2024-09-13T14:43:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:00 smithi042 ceph-mon[108570]: from='client.54589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:00.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.649 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:00.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:00 smithi159 ceph-mon[102519]: 1.7 deep-scrub starts 2024-09-13T14:43:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:00 smithi159 ceph-mon[102519]: 1.7 deep-scrub ok 2024-09-13T14:43:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:00 smithi159 ceph-mon[102519]: 1.8 scrub starts 2024-09-13T14:43:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:00 smithi159 ceph-mon[102519]: 1.8 scrub ok 2024-09-13T14:43:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:00 smithi159 ceph-mon[102519]: from='client.54589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:43:00.751 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:00.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:00.870 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:00.874 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:00.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:00.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:00.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:01.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:01.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:01.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:01.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:01.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:01.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:01.510 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:01.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 84479 84463 117.293 75.625 0.0120015 0.00852047 2024-09-13T14:43:01.594 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:01 smithi042 ceph-mon[108570]: pgmap v510: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:43:01.595 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:01 smithi042 ceph-mon[113926]: pgmap v510: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:43:01.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:01.647 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:01.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:01 smithi159 ceph-mon[102519]: pgmap v510: 99 pgs: 2 activating, 1 peering, 96 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-13T14:43:01.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:01.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:01.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:01.872 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:01.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:01.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:01.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:02.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:02.153 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:02.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:02.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:02.326 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:43:02.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:02.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:02.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:02.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:02.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 85259 85243 115.802 48.75 0.0078733 0.00862998 2024-09-13T14:43:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[108570]: 1.5 scrub starts 2024-09-13T14:43:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[108570]: 1.5 scrub ok 2024-09-13T14:43:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2155953819' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[113926]: 1.5 scrub starts 2024-09-13T14:43:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[113926]: 1.5 scrub ok 2024-09-13T14:43:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2155953819' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.694 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:02.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:02.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:02 smithi159 ceph-mon[102519]: 1.5 scrub starts 2024-09-13T14:43:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:02 smithi159 ceph-mon[102519]: 1.5 scrub ok 2024-09-13T14:43:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2155953819' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:43:02.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:02.831 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:02.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:02.976 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:02.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:03.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:03.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:03.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:03.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:03.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:03.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:03.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:03.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 86570 86554 115.082 81.9375 0.00869505 0.00868459 2024-09-13T14:43:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[108570]: pgmap v511: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.46k op/s 2024-09-13T14:43:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[108570]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:43:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[113926]: pgmap v511: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.46k op/s 2024-09-13T14:43:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:03 smithi042 ceph-mon[113926]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:43:03.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:03.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:03.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:03 smithi159 ceph-mon[102519]: pgmap v511: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 93 MiB/s wr, 1.46k op/s 2024-09-13T14:43:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:03 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:43:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:03 smithi159 ceph-mon[102519]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T14:43:03.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:03.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:03.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:03.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:03.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:03.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:03.976 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":509,"stamp":"2024-09-13T14:42:58.185347+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5044359502,"num_objects":76990,"num_object_clones":0,"num_object_copies":230970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76990,"num_whiteouts":0,"num_read":242,"num_read_kb":526,"num_write":77166,"num_write_kb":4927331,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":1389878,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78663,"ondisk_log_size":78663,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17744792,"kb_used_data":14754008,"kb_used_omap":28,"kb_used_meta":2990627,"kb_avail":732216424,"statfs":{"total":767960285184,"available":749789618176,"internally_reserved":0,"allocated":15108104192,"data_stored":15101991188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":3062402449},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,6,5,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":351505166,"num_objects":5365,"num_object_clones":0,"num_object_copies":16095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5365,"num_whiteouts":0,"num_read":-115,"num_read_kb":-233,"num_write":5307,"num_write_kb":342765,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-220341,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.412891"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792540+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:42:54.792540+0000","last_peered":"2024-09-13T14:42:54.792540+0000","last_clean":"2024-09-13T14:42:54.792540+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:42:54.792540+0000","last_undegraded":"2024-09-13T14:42:54.792540+0000","last_fullsized":"2024-09-13T14:42:54.792540+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":134,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.283643+0000","last_change":"2024-09-13T14:40:20.250118+0000","last_active":"2024-09-13T14:42:55.283643+0000","last_peered":"2024-09-13T14:42:55.283643+0000","last_clean":"2024-09-13T14:42:55.283643+0000","last_became_active":"2024-09-13T14:40:20.246066+0000","last_became_peered":"2024-09-13T14:40:20.246066+0000","last_unstale":"2024-09-13T14:42:55.283643+0000","last_undegraded":"2024-09-13T14:42:55.283643+0000","last_fullsized":"2024-09-13T14:42:55.283643+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:48:44.791098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":115,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.285722+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:42:55.285722+0000","last_peered":"2024-09-13T14:42:55.285722+0000","last_clean":"2024-09-13T14:42:55.285722+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:42:55.285722+0000","last_undegraded":"2024-09-13T14:42:55.285722+0000","last_fullsized":"2024-09-13T14:42:55.285722+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":143,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.779939+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:42:52.779939+0000","last_peered":"2024-09-13T14:42:52.779939+0000","last_clean":"2024-09-13T14:42:52.779939+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:42:52.779939+0000","last_undegraded":"2024-09-13T14:42:52.779939+0000","last_fullsized":"2024-09-13T14:42:52.779939+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":144,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782424+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:42:52.782424+0000","last_peered":"2024-09-13T14:42:52.782424+0000","last_clean":"2024-09-13T14:42:52.782424+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:42:52.782424+0000","last_undegraded":"2024-09-13T14:42:52.782424+0000","last_fullsized":"2024-09-13T14:42:52.782424+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.285850+0000","last_change":"2024-09-13T14:20:45.773610+0000","last_active":"2024-09-13T14:42:55.285850+0000","last_peered":"2024-09-13T14:42:55.285850+0000","last_clean":"2024-09-13T14:42:55.285850+0000","last_became_active":"2024-09-13T14:20:45.773268+0000","last_became_peered":"2024-09-13T14:20:45.773268+0000","last_unstale":"2024-09-13T14:42:55.285850+0000","last_undegraded":"2024-09-13T14:42:55.285850+0000","last_fullsized":"2024-09-13T14:42:55.285850+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:22:39.603388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":144,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.801419+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:42:54.801419+0000","last_peered":"2024-09-13T14:42:54.801419+0000","last_clean":"2024-09-13T14:42:54.801419+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:42:54.801419+0000","last_undegraded":"2024-09-13T14:42:54.801419+0000","last_fullsized":"2024-09-13T14:42:54.801419+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":125,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.278105+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:42:55.278105+0000","last_peered":"2024-09-13T14:42:55.278105+0000","last_clean":"2024-09-13T14:42:55.278105+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:42:55.278105+0000","last_undegraded":"2024-09-13T14:42:55.278105+0000","last_fullsized":"2024-09-13T14:42:55.278105+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"148'4656","reported_seq":4685,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:53.473836+0000","last_change":"2024-09-13T14:42:07.783733+0000","last_active":"2024-09-13T14:42:53.473836+0000","last_peered":"2024-09-13T14:42:53.473836+0000","last_clean":"2024-09-13T14:42:53.473836+0000","last_became_active":"2024-09-13T14:42:07.783466+0000","last_became_peered":"2024-09-13T14:42:07.783466+0000","last_unstale":"2024-09-13T14:42:53.473836+0000","last_undegraded":"2024-09-13T14:42:53.473836+0000","last_fullsized":"2024-09-13T14:42:53.473836+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4656,"log_dups_size":0,"ondisk_log_size":4656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:55:51.970990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305135616,"num_objects":4656,"num_object_clones":0,"num_object_copies":13968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4656,"num_write_kb":297984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":118,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.283006+0000","last_change":"2024-09-13T14:34:44.850556+0000","last_active":"2024-09-13T14:42:55.283006+0000","last_peered":"2024-09-13T14:42:55.283006+0000","last_clean":"2024-09-13T14:42:55.283006+0000","last_became_active":"2024-09-13T14:34:44.850147+0000","last_became_peered":"2024-09-13T14:34:44.850147+0000","last_unstale":"2024-09-13T14:42:55.283006+0000","last_undegraded":"2024-09-13T14:42:55.283006+0000","last_fullsized":"2024-09-13T14:42:55.283006+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:35.785947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":101,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.284820+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:42:55.284820+0000","last_peered":"2024-09-13T14:42:55.284820+0000","last_clean":"2024-09-13T14:42:55.284820+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:42:55.284820+0000","last_undegraded":"2024-09-13T14:42:55.284820+0000","last_fullsized":"2024-09-13T14:42:55.284820+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"152'4934","reported_seq":4971,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.637892+0000","last_change":"2024-09-13T14:42:07.780479+0000","last_active":"2024-09-13T14:42:57.637892+0000","last_peered":"2024-09-13T14:42:57.637892+0000","last_clean":"2024-09-13T14:42:57.637892+0000","last_became_active":"2024-09-13T14:42:07.780123+0000","last_became_peered":"2024-09-13T14:42:07.780123+0000","last_unstale":"2024-09-13T14:42:57.637892+0000","last_undegraded":"2024-09-13T14:42:57.637892+0000","last_fullsized":"2024-09-13T14:42:57.637892+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:47:20.035430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323354624,"num_objects":4934,"num_object_clones":0,"num_object_copies":14802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4934,"num_write_kb":315776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":119,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.282887+0000","last_change":"2024-09-13T14:35:58.847836+0000","last_active":"2024-09-13T14:42:55.282887+0000","last_peered":"2024-09-13T14:42:55.282887+0000","last_clean":"2024-09-13T14:42:55.282887+0000","last_became_active":"2024-09-13T14:35:58.847447+0000","last_became_peered":"2024-09-13T14:35:58.847447+0000","last_unstale":"2024-09-13T14:42:55.282887+0000","last_undegraded":"2024-09-13T14:42:55.282887+0000","last_fullsized":"2024-09-13T14:42:55.282887+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:16:54.473088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":131,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787010+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:42:54.787010+0000","last_peered":"2024-09-13T14:42:54.787010+0000","last_clean":"2024-09-13T14:42:54.787010+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:42:54.787010+0000","last_undegraded":"2024-09-13T14:42:54.787010+0000","last_fullsized":"2024-09-13T14:42:54.787010+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"152'4857","reported_seq":4879,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.525494+0000","last_change":"2024-09-13T14:42:07.781173+0000","last_active":"2024-09-13T14:42:56.525494+0000","last_peered":"2024-09-13T14:42:56.525494+0000","last_clean":"2024-09-13T14:42:56.525494+0000","last_became_active":"2024-09-13T14:42:07.780794+0000","last_became_peered":"2024-09-13T14:42:07.780794+0000","last_unstale":"2024-09-13T14:42:56.525494+0000","last_undegraded":"2024-09-13T14:42:56.525494+0000","last_fullsized":"2024-09-13T14:42:56.525494+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:03.397389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318308352,"num_objects":4857,"num_object_clones":0,"num_object_copies":14571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":310848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":103,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.285130+0000","last_change":"2024-09-13T14:20:45.769982+0000","last_active":"2024-09-13T14:42:55.285130+0000","last_peered":"2024-09-13T14:42:55.285130+0000","last_clean":"2024-09-13T14:42:55.285130+0000","last_became_active":"2024-09-13T14:20:45.769719+0000","last_became_peered":"2024-09-13T14:20:45.769719+0000","last_unstale":"2024-09-13T14:42:55.285130+0000","last_undegraded":"2024-09-13T14:42:55.285130+0000","last_fullsized":"2024-09-13T14:42:55.285130+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:21.285862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.278490+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:42:55.278490+0000","last_peered":"2024-09-13T14:42:55.278490+0000","last_clean":"2024-09-13T14:42:55.278490+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:42:55.278490+0000","last_undegraded":"2024-09-13T14:42:55.278490+0000","last_fullsized":"2024-09-13T14:42:55.278490+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"152'4737","reported_seq":4759,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.498916+0000","last_change":"2024-09-13T14:42:07.781817+0000","last_active":"2024-09-13T14:42:56.498916+0000","last_peered":"2024-09-13T14:42:56.498916+0000","last_clean":"2024-09-13T14:42:56.498916+0000","last_became_active":"2024-09-13T14:42:07.781534+0000","last_became_peered":"2024-09-13T14:42:07.781534+0000","last_unstale":"2024-09-13T14:42:56.498916+0000","last_undegraded":"2024-09-13T14:42:56.498916+0000","last_fullsized":"2024-09-13T14:42:56.498916+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:27.372521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310444032,"num_objects":4737,"num_object_clones":0,"num_object_copies":14211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4737,"num_write_kb":303168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":116,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.284431+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:42:55.284431+0000","last_peered":"2024-09-13T14:42:55.284431+0000","last_clean":"2024-09-13T14:42:55.284431+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:42:55.284431+0000","last_undegraded":"2024-09-13T14:42:55.284431+0000","last_fullsized":"2024-09-13T14:42:55.284431+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":163,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.781026+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:42:52.781026+0000","last_peered":"2024-09-13T14:42:52.781026+0000","last_clean":"2024-09-13T14:42:52.781026+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:42:52.781026+0000","last_undegraded":"2024-09-13T14:42:52.781026+0000","last_fullsized":"2024-09-13T14:42:52.781026+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":488,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.294839+0000","last_change":"2024-09-13T14:42:56.294839+0000","last_active":"2024-09-13T14:42:56.294839+0000","last_peered":"2024-09-13T14:42:56.294839+0000","last_clean":"2024-09-13T14:42:56.294839+0000","last_became_active":"2024-09-13T14:42:56.294361+0000","last_became_peered":"2024-09-13T14:42:56.294361+0000","last_unstale":"2024-09-13T14:42:56.294839+0000","last_undegraded":"2024-09-13T14:42:56.294839+0000","last_fullsized":"2024-09-13T14:42:56.294839+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:49:13.859283+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.1","version":"150'4730","reported_seq":4763,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.239168+0000","last_change":"2024-09-13T14:42:07.942573+0000","last_active":"2024-09-13T14:42:55.239168+0000","last_peered":"2024-09-13T14:42:55.239168+0000","last_clean":"2024-09-13T14:42:55.239168+0000","last_became_active":"2024-09-13T14:42:07.942229+0000","last_became_peered":"2024-09-13T14:42:07.942229+0000","last_unstale":"2024-09-13T14:42:55.239168+0000","last_undegraded":"2024-09-13T14:42:55.239168+0000","last_fullsized":"2024-09-13T14:42:55.239168+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4730,"log_dups_size":0,"ondisk_log_size":4730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:29:32.927262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4730,"num_write_kb":302720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792197+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:42:54.792197+0000","last_peered":"2024-09-13T14:42:54.792197+0000","last_clean":"2024-09-13T14:42:54.792197+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:42:54.792197+0000","last_undegraded":"2024-09-13T14:42:54.792197+0000","last_fullsized":"2024-09-13T14:42:54.792197+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":115,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.788310+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:42:54.788310+0000","last_peered":"2024-09-13T14:42:54.788310+0000","last_clean":"2024-09-13T14:42:54.788310+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:42:54.788310+0000","last_undegraded":"2024-09-13T14:42:54.788310+0000","last_fullsized":"2024-09-13T14:42:54.788310+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":374,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.297115+0000","last_change":"2024-09-13T14:42:56.296786+0000","last_active":"2024-09-13T14:42:56.297115+0000","last_peered":"2024-09-13T14:42:56.297115+0000","last_clean":"2024-09-13T14:42:56.297115+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:42:56.297115+0000","last_undegraded":"2024-09-13T14:42:56.297115+0000","last_fullsized":"2024-09-13T14:42:56.297115+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.0","version":"152'4921","reported_seq":4958,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:57.649738+0000","last_change":"2024-09-13T14:42:07.942557+0000","last_active":"2024-09-13T14:42:57.649738+0000","last_peered":"2024-09-13T14:42:57.649738+0000","last_clean":"2024-09-13T14:42:57.649738+0000","last_became_active":"2024-09-13T14:42:07.942184+0000","last_became_peered":"2024-09-13T14:42:07.942184+0000","last_unstale":"2024-09-13T14:42:57.649738+0000","last_undegraded":"2024-09-13T14:42:57.649738+0000","last_fullsized":"2024-09-13T14:42:57.649738+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4921,"log_dups_size":0,"ondisk_log_size":4921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:25.265169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322502656,"num_objects":4921,"num_object_clones":0,"num_object_copies":14763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4921,"num_write_kb":314944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":138,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.278670+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:42:55.278670+0000","last_peered":"2024-09-13T14:42:55.278670+0000","last_clean":"2024-09-13T14:42:55.278670+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:42:55.278670+0000","last_undegraded":"2024-09-13T14:42:55.278670+0000","last_fullsized":"2024-09-13T14:42:55.278670+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":130,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787485+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:42:54.787485+0000","last_peered":"2024-09-13T14:42:54.787485+0000","last_clean":"2024-09-13T14:42:54.787485+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:42:54.787485+0000","last_undegraded":"2024-09-13T14:42:54.787485+0000","last_fullsized":"2024-09-13T14:42:54.787485+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"152'4997","reported_seq":5019,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.294247+0000","last_change":"2024-09-13T14:42:07.780892+0000","last_active":"2024-09-13T14:42:56.294247+0000","last_peered":"2024-09-13T14:42:56.294247+0000","last_clean":"2024-09-13T14:42:56.294247+0000","last_became_active":"2024-09-13T14:42:07.780540+0000","last_became_peered":"2024-09-13T14:42:07.780540+0000","last_unstale":"2024-09-13T14:42:56.294247+0000","last_undegraded":"2024-09-13T14:42:56.294247+0000","last_fullsized":"2024-09-13T14:42:56.294247+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4997,"log_dups_size":0,"ondisk_log_size":4997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:27.427471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327483392,"num_objects":4997,"num_object_clones":0,"num_object_copies":14991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4997,"num_write_kb":319808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787086+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:42:54.787086+0000","last_peered":"2024-09-13T14:42:54.787086+0000","last_clean":"2024-09-13T14:42:54.787086+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:42:54.787086+0000","last_undegraded":"2024-09-13T14:42:54.787086+0000","last_fullsized":"2024-09-13T14:42:54.787086+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787875+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:42:54.787875+0000","last_peered":"2024-09-13T14:42:54.787875+0000","last_clean":"2024-09-13T14:42:54.787875+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:42:54.787875+0000","last_undegraded":"2024-09-13T14:42:54.787875+0000","last_fullsized":"2024-09-13T14:42:54.787875+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":374,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.290298+0000","last_change":"2024-09-13T14:42:56.289998+0000","last_active":"2024-09-13T14:42:56.290298+0000","last_peered":"2024-09-13T14:42:56.290298+0000","last_clean":"2024-09-13T14:42:56.290298+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:42:56.290298+0000","last_undegraded":"2024-09-13T14:42:56.290298+0000","last_fullsized":"2024-09-13T14:42:56.290298+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":74,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"150'4767","reported_seq":4800,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.238988+0000","last_change":"2024-09-13T14:42:07.780605+0000","last_active":"2024-09-13T14:42:55.238988+0000","last_peered":"2024-09-13T14:42:55.238988+0000","last_clean":"2024-09-13T14:42:55.238988+0000","last_became_active":"2024-09-13T14:42:07.780309+0000","last_became_peered":"2024-09-13T14:42:07.780309+0000","last_unstale":"2024-09-13T14:42:55.238988+0000","last_undegraded":"2024-09-13T14:42:55.238988+0000","last_fullsized":"2024-09-13T14:42:55.238988+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4767,"log_dups_size":0,"ondisk_log_size":4767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:29.840193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312410112,"num_objects":4767,"num_object_clones":0,"num_object_copies":14301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4767,"num_write_kb":305088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":121,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.793196+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:42:54.793196+0000","last_peered":"2024-09-13T14:42:54.793196+0000","last_clean":"2024-09-13T14:42:54.793196+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:42:54.793196+0000","last_undegraded":"2024-09-13T14:42:54.793196+0000","last_fullsized":"2024-09-13T14:42:54.793196+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":123,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.276930+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:42:56.276930+0000","last_peered":"2024-09-13T14:42:56.276930+0000","last_clean":"2024-09-13T14:42:56.276930+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:42:56.276930+0000","last_undegraded":"2024-09-13T14:42:56.276930+0000","last_fullsized":"2024-09-13T14:42:56.276930+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"150'4792","reported_seq":4825,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.211413+0000","last_change":"2024-09-13T14:42:07.782112+0000","last_active":"2024-09-13T14:42:55.211413+0000","last_peered":"2024-09-13T14:42:55.211413+0000","last_clean":"2024-09-13T14:42:55.211413+0000","last_became_active":"2024-09-13T14:42:07.781780+0000","last_became_peered":"2024-09-13T14:42:07.781780+0000","last_unstale":"2024-09-13T14:42:55.211413+0000","last_undegraded":"2024-09-13T14:42:55.211413+0000","last_fullsized":"2024-09-13T14:42:55.211413+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:07.810835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314048512,"num_objects":4792,"num_object_clones":0,"num_object_copies":14376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4792,"num_write_kb":306688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":471,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.282318+0000","last_change":"2024-09-13T14:42:50.766983+0000","last_active":"2024-09-13T14:42:55.282318+0000","last_peered":"2024-09-13T14:42:55.282318+0000","last_clean":"2024-09-13T14:42:55.282318+0000","last_became_active":"2024-09-13T14:42:47.952022+0000","last_became_peered":"2024-09-13T14:42:47.952022+0000","last_unstale":"2024-09-13T14:42:55.282318+0000","last_undegraded":"2024-09-13T14:42:55.282318+0000","last_fullsized":"2024-09-13T14:42:55.282318+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:50.766880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:50.766880+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:43.745219+0000","scrub_duration":0.0014951610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":131,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.284314+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:42:55.284314+0000","last_peered":"2024-09-13T14:42:55.284314+0000","last_clean":"2024-09-13T14:42:55.284314+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:42:55.284314+0000","last_undegraded":"2024-09-13T14:42:55.284314+0000","last_fullsized":"2024-09-13T14:42:55.284314+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792305+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:42:54.792305+0000","last_peered":"2024-09-13T14:42:54.792305+0000","last_clean":"2024-09-13T14:42:54.792305+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:42:54.792305+0000","last_undegraded":"2024-09-13T14:42:54.792305+0000","last_fullsized":"2024-09-13T14:42:54.792305+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":102,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.279000+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:42:55.279000+0000","last_peered":"2024-09-13T14:42:55.279000+0000","last_clean":"2024-09-13T14:42:55.279000+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:42:55.279000+0000","last_undegraded":"2024-09-13T14:42:55.279000+0000","last_fullsized":"2024-09-13T14:42:55.279000+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":383,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.684208+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:42:56.684208+0000","last_peered":"2024-09-13T14:42:56.684208+0000","last_clean":"2024-09-13T14:42:56.684208+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:42:56.684208+0000","last_undegraded":"2024-09-13T14:42:56.684208+0000","last_fullsized":"2024-09-13T14:42:56.684208+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.b","version":"148'4726","reported_seq":4755,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:53.478161+0000","last_change":"2024-09-13T14:42:07.942512+0000","last_active":"2024-09-13T14:42:53.478161+0000","last_peered":"2024-09-13T14:42:53.478161+0000","last_clean":"2024-09-13T14:42:53.478161+0000","last_became_active":"2024-09-13T14:42:07.942191+0000","last_became_peered":"2024-09-13T14:42:07.942191+0000","last_unstale":"2024-09-13T14:42:53.478161+0000","last_undegraded":"2024-09-13T14:42:53.478161+0000","last_fullsized":"2024-09-13T14:42:53.478161+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4726,"log_dups_size":0,"ondisk_log_size":4726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:05:07.538738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309723136,"num_objects":4726,"num_object_clones":0,"num_object_copies":14178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4726,"num_write_kb":302464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":146,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.282212+0000","last_change":"2024-09-13T14:35:58.848111+0000","last_active":"2024-09-13T14:42:55.282212+0000","last_peered":"2024-09-13T14:42:55.282212+0000","last_clean":"2024-09-13T14:42:55.282212+0000","last_became_active":"2024-09-13T14:35:58.847625+0000","last_became_peered":"2024-09-13T14:35:58.847625+0000","last_unstale":"2024-09-13T14:42:55.282212+0000","last_undegraded":"2024-09-13T14:42:55.282212+0000","last_fullsized":"2024-09-13T14:42:55.282212+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:48:15.257329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.780377+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:42:52.780377+0000","last_peered":"2024-09-13T14:42:52.780377+0000","last_clean":"2024-09-13T14:42:52.780377+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:42:52.780377+0000","last_undegraded":"2024-09-13T14:42:52.780377+0000","last_fullsized":"2024-09-13T14:42:52.780377+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":155,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782223+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:42:52.782223+0000","last_peered":"2024-09-13T14:42:52.782223+0000","last_clean":"2024-09-13T14:42:52.782223+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:42:52.782223+0000","last_undegraded":"2024-09-13T14:42:52.782223+0000","last_fullsized":"2024-09-13T14:42:52.782223+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":476,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.283824+0000","last_change":"2024-09-13T14:42:56.283824+0000","last_active":"2024-09-13T14:42:56.283824+0000","last_peered":"2024-09-13T14:42:56.283824+0000","last_clean":"2024-09-13T14:42:56.283824+0000","last_became_active":"2024-09-13T14:42:56.283454+0000","last_became_peered":"2024-09-13T14:42:56.283454+0000","last_unstale":"2024-09-13T14:42:56.283824+0000","last_undegraded":"2024-09-13T14:42:56.283824+0000","last_fullsized":"2024-09-13T14:42:56.283824+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.a","version":"152'4870","reported_seq":4892,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.509886+0000","last_change":"2024-09-13T14:42:07.781099+0000","last_active":"2024-09-13T14:42:56.509886+0000","last_peered":"2024-09-13T14:42:56.509886+0000","last_clean":"2024-09-13T14:42:56.509886+0000","last_became_active":"2024-09-13T14:42:07.780786+0000","last_became_peered":"2024-09-13T14:42:07.780786+0000","last_unstale":"2024-09-13T14:42:56.509886+0000","last_undegraded":"2024-09-13T14:42:56.509886+0000","last_fullsized":"2024-09-13T14:42:56.509886+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4870,"log_dups_size":0,"ondisk_log_size":4870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:24.082388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319160320,"num_objects":4870,"num_object_clones":0,"num_object_copies":14610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4870,"num_write_kb":311680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":145,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.283110+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:42:55.283110+0000","last_peered":"2024-09-13T14:42:55.283110+0000","last_clean":"2024-09-13T14:42:55.283110+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:42:55.283110+0000","last_undegraded":"2024-09-13T14:42:55.283110+0000","last_fullsized":"2024-09-13T14:42:55.283110+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.275794+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:42:56.275794+0000","last_peered":"2024-09-13T14:42:56.275794+0000","last_clean":"2024-09-13T14:42:56.275794+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:42:56.275794+0000","last_undegraded":"2024-09-13T14:42:56.275794+0000","last_fullsized":"2024-09-13T14:42:56.275794+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":158,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.276768+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:42:56.276768+0000","last_peered":"2024-09-13T14:42:56.276768+0000","last_clean":"2024-09-13T14:42:56.276768+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:42:56.276768+0000","last_undegraded":"2024-09-13T14:42:56.276768+0000","last_fullsized":"2024-09-13T14:42:56.276768+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":386,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.295979+0000","last_change":"2024-09-13T14:42:56.295979+0000","last_active":"2024-09-13T14:42:56.295979+0000","last_peered":"2024-09-13T14:42:56.295979+0000","last_clean":"2024-09-13T14:42:56.295979+0000","last_became_active":"2024-09-13T14:42:56.290769+0000","last_became_peered":"2024-09-13T14:42:56.290769+0000","last_unstale":"2024-09-13T14:42:56.295979+0000","last_undegraded":"2024-09-13T14:42:56.295979+0000","last_fullsized":"2024-09-13T14:42:56.295979+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:27.386430+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"148'4669","reported_seq":4697,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:53.486739+0000","last_change":"2024-09-13T14:42:07.783147+0000","last_active":"2024-09-13T14:42:53.486739+0000","last_peered":"2024-09-13T14:42:53.486739+0000","last_clean":"2024-09-13T14:42:53.486739+0000","last_became_active":"2024-09-13T14:42:07.782831+0000","last_became_peered":"2024-09-13T14:42:07.782831+0000","last_unstale":"2024-09-13T14:42:53.486739+0000","last_undegraded":"2024-09-13T14:42:53.486739+0000","last_fullsized":"2024-09-13T14:42:53.486739+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4669,"log_dups_size":0,"ondisk_log_size":4669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:03.638364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305987584,"num_objects":4669,"num_object_clones":0,"num_object_copies":14007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4669,"num_write_kb":298816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":132,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.285807+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:42:55.285807+0000","last_peered":"2024-09-13T14:42:55.285807+0000","last_clean":"2024-09-13T14:42:55.285807+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:42:55.285807+0000","last_undegraded":"2024-09-13T14:42:55.285807+0000","last_fullsized":"2024-09-13T14:42:55.285807+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":43,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792017+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:42:54.792017+0000","last_peered":"2024-09-13T14:42:54.792017+0000","last_clean":"2024-09-13T14:42:54.792017+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:42:54.792017+0000","last_undegraded":"2024-09-13T14:42:54.792017+0000","last_fullsized":"2024-09-13T14:42:54.792017+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":134,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.283836+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:42:56.283836+0000","last_peered":"2024-09-13T14:42:56.283836+0000","last_clean":"2024-09-13T14:42:56.283836+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:42:56.283836+0000","last_undegraded":"2024-09-13T14:42:56.283836+0000","last_fullsized":"2024-09-13T14:42:56.283836+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":375,"reported_epoch":152,"state":"activating","last_fresh":"2024-09-13T14:42:56.275326+0000","last_change":"2024-09-13T14:42:56.275326+0000","last_active":"2024-09-13T14:42:55.279582+0000","last_peered":"2024-09-13T14:42:54.802239+0000","last_clean":"2024-09-13T14:28:22.829429+0000","last_became_active":"2024-09-13T14:42:54.801922+0000","last_became_peered":"2024-09-13T14:42:54.801922+0000","last_unstale":"2024-09-13T14:42:56.275326+0000","last_undegraded":"2024-09-13T14:42:56.275326+0000","last_fullsized":"2024-09-13T14:42:56.275326+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":["6","0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.f","version":"152'4769","reported_seq":4791,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.513232+0000","last_change":"2024-09-13T14:42:07.781837+0000","last_active":"2024-09-13T14:42:56.513232+0000","last_peered":"2024-09-13T14:42:56.513232+0000","last_clean":"2024-09-13T14:42:56.513232+0000","last_became_active":"2024-09-13T14:42:07.781584+0000","last_became_peered":"2024-09-13T14:42:07.781584+0000","last_unstale":"2024-09-13T14:42:56.513232+0000","last_undegraded":"2024-09-13T14:42:56.513232+0000","last_fullsized":"2024-09-13T14:42:56.513232+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4769,"log_dups_size":0,"ondisk_log_size":4769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:21.323962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312541184,"num_objects":4769,"num_object_clones":0,"num_object_copies":14307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4769,"num_write_kb":305216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":132,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.286091+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:42:55.286091+0000","last_peered":"2024-09-13T14:42:55.286091+0000","last_clean":"2024-09-13T14:42:55.286091+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:42:55.286091+0000","last_undegraded":"2024-09-13T14:42:55.286091+0000","last_fullsized":"2024-09-13T14:42:55.286091+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":172,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.284034+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:42:56.284034+0000","last_peered":"2024-09-13T14:42:56.284034+0000","last_clean":"2024-09-13T14:42:56.284034+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:42:56.284034+0000","last_undegraded":"2024-09-13T14:42:56.284034+0000","last_fullsized":"2024-09-13T14:42:56.284034+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":42,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.280659+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:42:56.280659+0000","last_peered":"2024-09-13T14:42:56.280659+0000","last_clean":"2024-09-13T14:42:56.280659+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:42:56.280659+0000","last_undegraded":"2024-09-13T14:42:56.280659+0000","last_fullsized":"2024-09-13T14:42:56.280659+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":375,"reported_epoch":152,"state":"activating","last_fresh":"2024-09-13T14:42:56.275633+0000","last_change":"2024-09-13T14:42:56.275633+0000","last_active":"2024-09-13T14:42:55.279410+0000","last_peered":"2024-09-13T14:42:54.802166+0000","last_clean":"2024-09-13T14:28:22.829429+0000","last_became_active":"2024-09-13T14:42:54.801876+0000","last_became_peered":"2024-09-13T14:42:54.801876+0000","last_unstale":"2024-09-13T14:42:56.275633+0000","last_undegraded":"2024-09-13T14:42:56.275633+0000","last_fullsized":"2024-09-13T14:42:56.275633+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.e","version":"152'4833","reported_seq":4855,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.307879+0000","last_change":"2024-09-13T14:42:07.781001+0000","last_active":"2024-09-13T14:42:56.307879+0000","last_peered":"2024-09-13T14:42:56.307879+0000","last_clean":"2024-09-13T14:42:56.307879+0000","last_became_active":"2024-09-13T14:42:07.780697+0000","last_became_peered":"2024-09-13T14:42:07.780697+0000","last_unstale":"2024-09-13T14:42:56.307879+0000","last_undegraded":"2024-09-13T14:42:56.307879+0000","last_fullsized":"2024-09-13T14:42:56.307879+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4833,"log_dups_size":0,"ondisk_log_size":4833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:29:01.574906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316735488,"num_objects":4833,"num_object_clones":0,"num_object_copies":14499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4833,"num_write_kb":309312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.283943+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:42:56.283943+0000","last_peered":"2024-09-13T14:42:56.283943+0000","last_clean":"2024-09-13T14:42:56.283943+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:42:56.283943+0000","last_undegraded":"2024-09-13T14:42:56.283943+0000","last_fullsized":"2024-09-13T14:42:56.283943+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":131,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.285230+0000","last_change":"2024-09-13T14:34:44.823431+0000","last_active":"2024-09-13T14:42:55.285230+0000","last_peered":"2024-09-13T14:42:55.285230+0000","last_clean":"2024-09-13T14:42:55.285230+0000","last_became_active":"2024-09-13T14:34:44.823118+0000","last_became_peered":"2024-09-13T14:34:44.823118+0000","last_unstale":"2024-09-13T14:42:55.285230+0000","last_undegraded":"2024-09-13T14:42:55.285230+0000","last_fullsized":"2024-09-13T14:42:55.285230+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:40:49.072518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782438+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:42:52.782438+0000","last_peered":"2024-09-13T14:42:52.782438+0000","last_clean":"2024-09-13T14:42:52.782438+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:42:52.782438+0000","last_undegraded":"2024-09-13T14:42:52.782438+0000","last_fullsized":"2024-09-13T14:42:52.782438+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":365,"reported_epoch":151,"state":"peering","last_fresh":"2024-09-13T14:42:54.800171+0000","last_change":"2024-09-13T14:42:56.257060+0000","last_active":"2024-09-13T14:42:56.257060+0000","last_peered":"2024-09-13T14:42:54.800171+0000","last_clean":"2024-09-13T14:28:22.829429+0000","last_became_active":"2024-09-13T14:42:54.799790+0000","last_became_peered":"2024-09-13T14:42:54.799790+0000","last_unstale":"2024-09-13T14:42:54.800171+0000","last_undegraded":"2024-09-13T14:42:54.800171+0000","last_fullsized":"2024-09-13T14:42:54.782932+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.d","version":"151'4860","reported_seq":4882,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.254144+0000","last_change":"2024-09-13T14:42:07.782820+0000","last_active":"2024-09-13T14:42:56.254144+0000","last_peered":"2024-09-13T14:42:56.254144+0000","last_clean":"2024-09-13T14:42:56.254144+0000","last_became_active":"2024-09-13T14:42:07.782375+0000","last_became_peered":"2024-09-13T14:42:07.782375+0000","last_unstale":"2024-09-13T14:42:56.254144+0000","last_undegraded":"2024-09-13T14:42:56.254144+0000","last_fullsized":"2024-09-13T14:42:56.254144+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4860,"log_dups_size":0,"ondisk_log_size":4860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:39.823277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318504960,"num_objects":4860,"num_object_clones":0,"num_object_copies":14580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4860,"num_write_kb":311040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":107,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.801537+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:42:54.801537+0000","last_peered":"2024-09-13T14:42:54.801537+0000","last_clean":"2024-09-13T14:42:54.801537+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:42:54.801537+0000","last_undegraded":"2024-09-13T14:42:54.801537+0000","last_fullsized":"2024-09-13T14:42:54.801537+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":122,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.280401+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:42:56.280401+0000","last_peered":"2024-09-13T14:42:56.280401+0000","last_clean":"2024-09-13T14:42:56.280401+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:42:56.280401+0000","last_undegraded":"2024-09-13T14:42:56.280401+0000","last_fullsized":"2024-09-13T14:42:56.280401+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792550+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:42:54.792550+0000","last_peered":"2024-09-13T14:42:54.792550+0000","last_clean":"2024-09-13T14:42:54.792550+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:42:54.792550+0000","last_undegraded":"2024-09-13T14:42:54.792550+0000","last_fullsized":"2024-09-13T14:42:54.792550+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":454,"reported_epoch":147,"state":"active+clean","last_fresh":"2024-09-13T14:42:47.960142+0000","last_change":"2024-09-13T14:42:47.958856+0000","last_active":"2024-09-13T14:42:47.960142+0000","last_peered":"2024-09-13T14:42:47.960142+0000","last_clean":"2024-09-13T14:42:47.960142+0000","last_became_active":"2024-09-13T14:42:47.958425+0000","last_became_peered":"2024-09-13T14:42:47.958425+0000","last_unstale":"2024-09-13T14:42:47.960142+0000","last_undegraded":"2024-09-13T14:42:47.960142+0000","last_fullsized":"2024-09-13T14:42:47.960142+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:17:00.645661+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"150'4847","reported_seq":4878,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.248407+0000","last_change":"2024-09-13T14:42:07.782967+0000","last_active":"2024-09-13T14:42:55.248407+0000","last_peered":"2024-09-13T14:42:55.248407+0000","last_clean":"2024-09-13T14:42:55.248407+0000","last_became_active":"2024-09-13T14:42:07.782365+0000","last_became_peered":"2024-09-13T14:42:07.782365+0000","last_unstale":"2024-09-13T14:42:55.248407+0000","last_undegraded":"2024-09-13T14:42:55.248407+0000","last_fullsized":"2024-09-13T14:42:55.248407+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:02.386475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317652992,"num_objects":4847,"num_object_clones":0,"num_object_copies":14541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4847,"num_write_kb":310208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":147,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782519+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:42:52.782519+0000","last_peered":"2024-09-13T14:42:52.782519+0000","last_clean":"2024-09-13T14:42:52.782519+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:42:52.782519+0000","last_undegraded":"2024-09-13T14:42:52.782519+0000","last_fullsized":"2024-09-13T14:42:52.782519+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":130,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.279345+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:42:55.279345+0000","last_peered":"2024-09-13T14:42:55.279345+0000","last_clean":"2024-09-13T14:42:55.279345+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:42:55.279345+0000","last_undegraded":"2024-09-13T14:42:55.279345+0000","last_fullsized":"2024-09-13T14:42:55.279345+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":101,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.786946+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:42:54.786946+0000","last_peered":"2024-09-13T14:42:54.786946+0000","last_clean":"2024-09-13T14:42:54.786946+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:42:54.786946+0000","last_undegraded":"2024-09-13T14:42:54.786946+0000","last_fullsized":"2024-09-13T14:42:54.786946+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":115,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787253+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:42:54.787253+0000","last_peered":"2024-09-13T14:42:54.787253+0000","last_clean":"2024-09-13T14:42:54.787253+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:42:54.787253+0000","last_undegraded":"2024-09-13T14:42:54.787253+0000","last_fullsized":"2024-09-13T14:42:54.787253+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":124,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.788454+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:42:54.788454+0000","last_peered":"2024-09-13T14:42:54.788454+0000","last_clean":"2024-09-13T14:42:54.788454+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:42:54.788454+0000","last_undegraded":"2024-09-13T14:42:54.788454+0000","last_fullsized":"2024-09-13T14:42:54.788454+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.277319+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:42:56.277319+0000","last_peered":"2024-09-13T14:42:56.277319+0000","last_clean":"2024-09-13T14:42:56.277319+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:42:56.277319+0000","last_undegraded":"2024-09-13T14:42:56.277319+0000","last_fullsized":"2024-09-13T14:42:56.277319+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":66,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.283693+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:42:55.283693+0000","last_peered":"2024-09-13T14:42:55.283693+0000","last_clean":"2024-09-13T14:42:55.283693+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:42:55.283693+0000","last_undegraded":"2024-09-13T14:42:55.283693+0000","last_fullsized":"2024-09-13T14:42:55.283693+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":116,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.284249+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:42:55.284249+0000","last_peered":"2024-09-13T14:42:55.284249+0000","last_clean":"2024-09-13T14:42:55.284249+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:42:55.284249+0000","last_undegraded":"2024-09-13T14:42:55.284249+0000","last_fullsized":"2024-09-13T14:42:55.284249+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":106,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.796077+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:42:54.796077+0000","last_peered":"2024-09-13T14:42:54.796077+0000","last_clean":"2024-09-13T14:42:54.796077+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:42:54.796077+0000","last_undegraded":"2024-09-13T14:42:54.796077+0000","last_fullsized":"2024-09-13T14:42:54.796077+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":142,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.787088+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:42:52.787088+0000","last_peered":"2024-09-13T14:42:52.787088+0000","last_clean":"2024-09-13T14:42:52.787088+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:42:52.787088+0000","last_undegraded":"2024-09-13T14:42:52.787088+0000","last_fullsized":"2024-09-13T14:42:52.787088+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":119,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792501+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:42:54.792501+0000","last_peered":"2024-09-13T14:42:54.792501+0000","last_clean":"2024-09-13T14:42:54.792501+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:42:54.792501+0000","last_undegraded":"2024-09-13T14:42:54.792501+0000","last_fullsized":"2024-09-13T14:42:54.792501+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.279249+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:42:55.279249+0000","last_peered":"2024-09-13T14:42:55.279249+0000","last_clean":"2024-09-13T14:42:55.279249+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:42:55.279249+0000","last_undegraded":"2024-09-13T14:42:55.279249+0000","last_fullsized":"2024-09-13T14:42:55.279249+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":134,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.793107+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:42:54.793107+0000","last_peered":"2024-09-13T14:42:54.793107+0000","last_clean":"2024-09-13T14:42:54.793107+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:42:54.793107+0000","last_undegraded":"2024-09-13T14:42:54.793107+0000","last_fullsized":"2024-09-13T14:42:54.793107+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":145,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782336+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:42:52.782336+0000","last_peered":"2024-09-13T14:42:52.782336+0000","last_clean":"2024-09-13T14:42:52.782336+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:42:52.782336+0000","last_undegraded":"2024-09-13T14:42:52.782336+0000","last_fullsized":"2024-09-13T14:42:52.782336+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":120,"reported_epoch":152,"state":"active+clean","last_fresh":"2024-09-13T14:42:56.275923+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:42:56.275923+0000","last_peered":"2024-09-13T14:42:56.275923+0000","last_clean":"2024-09-13T14:42:56.275923+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:42:56.275923+0000","last_undegraded":"2024-09-13T14:42:56.275923+0000","last_fullsized":"2024-09-13T14:42:56.275923+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":118,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792045+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:42:54.792045+0000","last_peered":"2024-09-13T14:42:54.792045+0000","last_clean":"2024-09-13T14:42:54.792045+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:42:54.792045+0000","last_undegraded":"2024-09-13T14:42:54.792045+0000","last_fullsized":"2024-09-13T14:42:54.792045+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":42,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792198+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:42:54.792198+0000","last_peered":"2024-09-13T14:42:54.792198+0000","last_clean":"2024-09-13T14:42:54.792198+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:42:54.792198+0000","last_undegraded":"2024-09-13T14:42:54.792198+0000","last_fullsized":"2024-09-13T14:42:54.792198+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":146,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.283358+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:42:55.283358+0000","last_peered":"2024-09-13T14:42:55.283358+0000","last_clean":"2024-09-13T14:42:55.283358+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:42:55.283358+0000","last_undegraded":"2024-09-13T14:42:55.283358+0000","last_fullsized":"2024-09-13T14:42:55.283358+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":148,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.782527+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:42:52.782527+0000","last_peered":"2024-09-13T14:42:52.782527+0000","last_clean":"2024-09-13T14:42:52.782527+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:42:52.782527+0000","last_undegraded":"2024-09-13T14:42:52.782527+0000","last_fullsized":"2024-09-13T14:42:52.782527+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":129,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787136+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:42:54.787136+0000","last_peered":"2024-09-13T14:42:54.787136+0000","last_clean":"2024-09-13T14:42:54.787136+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:42:54.787136+0000","last_undegraded":"2024-09-13T14:42:54.787136+0000","last_fullsized":"2024-09-13T14:42:54.787136+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":117,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.284322+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:42:55.284322+0000","last_peered":"2024-09-13T14:42:55.284322+0000","last_clean":"2024-09-13T14:42:55.284322+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:42:55.284322+0000","last_undegraded":"2024-09-13T14:42:55.284322+0000","last_fullsized":"2024-09-13T14:42:55.284322+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.787226+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:42:54.787226+0000","last_peered":"2024-09-13T14:42:54.787226+0000","last_clean":"2024-09-13T14:42:54.787226+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:42:54.787226+0000","last_undegraded":"2024-09-13T14:42:54.787226+0000","last_fullsized":"2024-09-13T14:42:54.787226+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":102,"reported_epoch":151,"state":"active+clean","last_fresh":"2024-09-13T14:42:55.286155+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:42:55.286155+0000","last_peered":"2024-09-13T14:42:55.286155+0000","last_clean":"2024-09-13T14:42:55.286155+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:42:55.286155+0000","last_undegraded":"2024-09-13T14:42:55.286155+0000","last_fullsized":"2024-09-13T14:42:55.286155+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":150,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.801327+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:42:54.801327+0000","last_peered":"2024-09-13T14:42:54.801327+0000","last_clean":"2024-09-13T14:42:54.801327+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:42:54.801327+0000","last_undegraded":"2024-09-13T14:42:54.801327+0000","last_fullsized":"2024-09-13T14:42:54.801327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":140,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.792493+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:42:54.792493+0000","last_peered":"2024-09-13T14:42:54.792493+0000","last_clean":"2024-09-13T14:42:54.792493+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:42:54.792493+0000","last_undegraded":"2024-09-13T14:42:54.792493+0000","last_fullsized":"2024-09-13T14:42:54.792493+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":150,"state":"active+clean","last_fresh":"2024-09-13T14:42:54.786859+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:42:54.786859+0000","last_peered":"2024-09-13T14:42:54.786859+0000","last_clean":"2024-09-13T14:42:54.786859+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:42:54.786859+0000","last_undegraded":"2024-09-13T14:42:54.786859+0000","last_fullsized":"2024-09-13T14:42:54.786859+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":148,"reported_epoch":148,"state":"active+clean","last_fresh":"2024-09-13T14:42:52.786959+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:42:52.786959+0000","last_peered":"2024-09-13T14:42:52.786959+0000","last_clean":"2024-09-13T14:42:52.786959+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:42:52.786959+0000","last_undegraded":"2024-09-13T14:42:52.786959+0000","last_fullsized":"2024-09-13T14:42:52.786959+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":5043978240,"num_objects":76965,"num_object_clones":0,"num_object_copies":230895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76965,"num_write_kb":4925760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15234367488,"data_stored":15234367488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76965,"ondisk_log_size":76965,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":124,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":375689,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":526,"num_write":137,"num_write_kb":1538,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1389754,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2969600,"data_stored":2951840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234274,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3328404,"kb_used_data":2791944,"kb_used_omap":1,"kb_used_meta":536446,"kb_avail":90416748,"statfs":{"total":95995035648,"available":92586749952,"internally_reserved":0,"allocated":2858950656,"data_stored":2858172560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":549321163},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495912,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433428,"kb_used_data":928964,"kb_used_omap":1,"kb_used_meta":504446,"kb_avail":92311724,"statfs":{"total":95995035648,"available":94527205376,"internally_reserved":0,"allocated":951259136,"data_stored":950485099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":516553159},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757550,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3495644,"kb_used_data":2792644,"kb_used_omap":1,"kb_used_meta":702974,"kb_avail":90249508,"statfs":{"total":95995035648,"available":92415496192,"internally_reserved":0,"allocated":2859667456,"data_stored":2858897364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":719845830},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019186,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505172,"kb_used_data":928512,"kb_used_omap":1,"kb_used_meta":576638,"kb_avail":92239980,"statfs":{"total":95995035648,"available":94453739520,"internally_reserved":0,"allocated":950796288,"data_stored":950025826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":590477766},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650398,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2531808,"kb_used_data":2405972,"kb_used_omap":4,"kb_used_meta":125819,"kb_avail":91213344,"statfs":{"total":95995035648,"available":93402464256,"internally_reserved":0,"allocated":2463715328,"data_stored":2462967804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":128839215},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879333,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":985020,"kb_used_data":905584,"kb_used_omap":6,"kb_used_meta":79417,"kb_avail":92760132,"statfs":{"total":95995035648,"available":94986375168,"internally_reserved":0,"allocated":927318016,"data_stored":926551696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":81323577},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435350,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3048908,"kb_used_data":2804344,"kb_used_omap":5,"kb_used_meta":244538,"kb_avail":90696244,"statfs":{"total":95995035648,"available":92872953856,"internally_reserved":0,"allocated":2871648256,"data_stored":2870889087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":250407265},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925924,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1416408,"kb_used_data":1196044,"kb_used_omap":5,"kb_used_meta":220346,"kb_avail":92328744,"statfs":{"total":95995035648,"available":94544633856,"internally_reserved":0,"allocated":1224749056,"data_stored":1224001752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":225634474},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1225326592,"data_stored":1225326592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2919563264,"data_stored":2919563264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":929169408,"data_stored":929169408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2505506816,"data_stored":2505506816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":954793984,"data_stored":954793984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2868051968,"data_stored":2868051968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":955252736,"data_stored":955252736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2876702720,"data_stored":2876702720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:43:03.981 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-09-13T14:43:04.073 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:04.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:04.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:04.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:04.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:04.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:04.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:04.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:04.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:04.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:04.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:04.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:04.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 87839 87823 114.336 79.3125 0.00495951 0.00874088 2024-09-13T14:43:04.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:04.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:04.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:04.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:04.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:04.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:04.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:05.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:05.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:05.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:05.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:05.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:05.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:05.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:05.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:05.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:05.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:05.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 89125 89109 113.643 80.375 0.00565387 0.00879405 2024-09-13T14:43:05.576 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:05 smithi042 ceph-mon[113926]: pgmap v513: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:43:05.576 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:05 smithi042 ceph-mon[108570]: pgmap v513: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:43:05.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.670 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:05.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:05.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:05 smithi159 ceph-mon[102519]: pgmap v513: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-13T14:43:05.771 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:05.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:05.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:05.933 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:05.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:05.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:06.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:06.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:06.212 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:06.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:06.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.414 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:06.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:06.515 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:06.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:06.576 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 90323 90307 112.868 74.875 0.0161594 0.00885494 2024-09-13T14:43:06.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:06.694 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:06.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:06.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.826 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:06.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:06.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:06.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:06.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:06.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:06.967 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:06.967 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:06.967 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:07.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:07.100 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:43:07.100 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:07.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:07.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:07.144 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:43:07.145 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:07.179 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:43:07.180 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:07.225 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:43:07.225 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:07.228 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:07.257 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:43:07.257 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:07.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:07.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:07.290 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:43:07.291 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:07.328 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:43:07.328 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:07.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:07.363 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:43:07.363 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:07.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:07.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:07.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:07.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:07.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:07.574 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 91458 91442 112.046 70.9375 0.00735801 0.00891973 2024-09-13T14:43:07.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:07.679 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:07.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:07.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:07.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:07.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:07.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:07.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:08.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:07 smithi042 ceph-mon[113926]: pgmap v514: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.25k op/s 2024-09-13T14:43:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1413745991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:07 smithi042 ceph-mon[108570]: pgmap v514: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.25k op/s 2024-09-13T14:43:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1413745991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:08.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:43:08.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:08.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:07 smithi159 ceph-mon[102519]: pgmap v514: 99 pgs: 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.25k op/s 2024-09-13T14:43:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1413745991' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:08.337 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:08.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:08.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:08.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:08.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 92494 92478 111.136 64.75 0.00298736 0.00898836 2024-09-13T14:43:08.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:08.620 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:08.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:08.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:08.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:08.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:08.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:08.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:09.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:09.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:09.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.130 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.231 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:09.269 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:09.371 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:09.400 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:09.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:09.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:09.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 93612 93596 110.357 69.875 0.00888017 0.00905616 2024-09-13T14:43:09.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:09.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:09.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:09.790 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:09.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:09.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:09.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:10.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:10.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:09 smithi042 ceph-mon[113926]: pgmap v515: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.21k op/s 2024-09-13T14:43:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:09 smithi042 ceph-mon[108570]: pgmap v515: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.21k op/s 2024-09-13T14:43:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:09] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-09-13T14:43:10.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:10.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:09 smithi159 ceph-mon[102519]: pgmap v515: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.21k op/s 2024-09-13T14:43:10.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:10.347 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:10.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:10.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:10.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:10.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 94499 94483 109.34 55.4375 0.0111545 0.00914058 2024-09-13T14:43:10.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:10.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:10.727 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:10.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:10.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:10.863 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:10.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:10.900 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:10.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.000 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:11.032 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:11.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:11.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:11.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.282 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:11.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:11.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:11.459 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:11.561 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:11.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 15 95605 95590 108.61 69.1875 0.0110848 0.00920296 2024-09-13T14:43:11.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:11.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.728 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86306 8.7 8.819b7977 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850244+0000", 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.11162264799998, 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99270645400000002, 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.729 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86306 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850244+0000", 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850244+0000", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850246+0000", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.733e-06 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850345+0000", 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.912e-05 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850516+0000", 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017110899999999999 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850526+0000", 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0664e-05 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.873935+0000", 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023408227 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.873946+0000", 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1547e-05 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.842939+0000", 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96899323999999998 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.733 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.842950+0000", 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0814e-05 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86312 8.7 8.d7aa377 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850543+0000", 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.11132301100002, 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99247957600000003, 2024-09-13T14:43:11.734 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86312 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850543+0000", 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.735 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850543+0000", 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850544+0000", 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8199999999999999e-07 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.736 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850603+0000", 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8950999999999999e-05 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850614+0000", 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0494e-05 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.737 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850621+0000", 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5259999999999996e-06 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.873970+0000", 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023348495 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.873976+0000", 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5490000000000003e-06 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.738 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.843013+0000", 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.969036919 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.843023+0000", 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9599999999999995e-06 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.739 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86322 8.7 8.37710417 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850631+0000", 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.111235212, 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99242394700000003, 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86322 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.740 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850631+0000", 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850631+0000", 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.741 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850632+0000", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4199999999999996e-07 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850682+0000", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9938999999999997e-05 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850691+0000", 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5109999999999992e-06 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.742 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850698+0000", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.511e-06 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.873998+0000", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023300042999999999 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.874004+0000", 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.63e-06 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.743 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.843046+0000", 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96904174499999995 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.843055+0000", 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0259999999999996e-06 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.744 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86323 8.a 8.fbbee2a (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850708+0000", 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.11115814300001, 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99602806499999996, 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86323 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.745 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850708+0000", 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850708+0000", 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.746 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.747 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.747 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850709+0000", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3600000000000004e-07 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850754+0000", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5117999999999997e-05 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850762+0000", 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8809999999999996e-06 2024-09-13T14:43:11.748 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850768+0000", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.083e-06 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.878687+0000", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.027918802999999999 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.878696+0000", 2024-09-13T14:43:11.749 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.789e-06 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.846716+0000", 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96802016800000001 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.846736+0000", 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0687e-05 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.750 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86325 8.3 8.153ea003 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850778+0000", 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.11108877200002, 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98931809900000001, 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86325 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.751 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850778+0000", 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850778+0000", 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.752 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850778+0000", 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7199999999999999e-07 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850824+0000", 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5986000000000003e-05 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850833+0000", 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7159999999999992e-06 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.753 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850839+0000", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4749999999999998e-06 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.852120+0000", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0012804520000000001 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.852133+0000", 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3424999999999999e-05 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.754 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.840083+0000", 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98794937000000005 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.840096+0000", 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3203e-05 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.755 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:86333 8.1 8.9161f5b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e109)", 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:35:57.850860+0000", 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 429.11100613100001, 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.989551862, 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86333 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.756 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850860+0000", 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850860+0000", 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.757 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850861+0000", 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7300000000000001e-07 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850909+0000", 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7933000000000003e-05 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850918+0000", 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3519999999999996e-06 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.758 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.850925+0000", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4520000000000002e-06 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.867123+0000", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016198489999999999 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:57.867133+0000", 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8109999999999997e-06 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.759 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.840401+0000", 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97326775300000001 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:35:58.840412+0000", 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1598e-05 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.760 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:121416 8.a 8:521c73d4:::benchmark_data_smithi042_130700_object121415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 387.72128089199998, 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.008701308, 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 121416 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.761 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240588+0000", 2024-09-13T14:43:11.762 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2309999999999999e-06 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240617+0000", 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9275e-05 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240638+0000", 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1373e-05 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.763 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240650+0000", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1075e-05 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240760+0000", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000110196 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240772+0000", 2024-09-13T14:43:11.764 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1838000000000001e-05 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.231939+0000", 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99116713999999995 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.232079+0000", 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013983299999999999 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.765 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.232195+0000", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116772 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.238677+0000", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064812339999999998 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.243060+0000", 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043835360000000004 2024-09-13T14:43:11.766 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249201+0000", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061413140000000001 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249238+0000", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7011000000000002e-05 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249287+0000", 2024-09-13T14:43:11.767 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8479999999999997e-05 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:121422 8.a 8:52568a6a:::benchmark_data_smithi042_130700_object121421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 387.71126416800001, 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0054267990000001, 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.768 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 121422 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:43:11.769 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250604+0000", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.72e-06 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250630+0000", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6095e-05 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250645+0000", 2024-09-13T14:43:11.770 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4562999999999999e-05 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250654+0000", 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3559999999999999e-06 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.254323+0000", 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003669285 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.771 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.254334+0000", 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0791999999999999e-05 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234556+0000", 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98022220000000004 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234637+0000", 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1094000000000007e-05 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.772 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:43:11.773 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234736+0000", 2024-09-13T14:43:11.773 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8134999999999995e-05 2024-09-13T14:43:11.776 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.776 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.776 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.243017+0000", 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082817710000000003 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249358+0000", 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0063408120000000004 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.777 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.255957+0000", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0065990629999999996 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.255995+0000", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7732999999999999e-05 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.256029+0000", 2024-09-13T14:43:11.778 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4180000000000001e-05 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187657 8.3 8:c5d07f4c:::benchmark_data_smithi042_130700_object63068:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.09883518499998, 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0351328040000001, 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.779 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187657 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.780 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863032+0000", 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0920000000000001e-06 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.781 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863036+0000", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.946e-06 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863059+0000", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2549e-05 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863068+0000", 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1050000000000001e-06 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.782 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864184+0000", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001115935 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864193+0000", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9369999999999993e-06 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898090+0000", 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0338969140000001 2024-09-13T14:43:11.783 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898164+0000", 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4326000000000006e-05 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187670 8.3 8:c61da759:::benchmark_data_smithi042_130700_object63081:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.09878620699999, 2024-09-13T14:43:11.784 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.035638155, 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187670 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:43:11.785 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863081+0000", 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1200000000000002e-07 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.786 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863084+0000", 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5329999999999999e-06 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863092+0000", 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7710000000000006e-06 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863098+0000", 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3730000000000001e-06 2024-09-13T14:43:11.787 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864683+0000", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001584963 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864691+0000", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1519999999999992e-06 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898675+0000", 2024-09-13T14:43:11.788 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.03398452 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898718+0000", 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3130999999999998e-05 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187680 8.3 8:ce9e033c:::benchmark_data_smithi042_130700_object63091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:43:11.789 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.09730624999997, 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0633122770000001, 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187680 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:43:11.790 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864562+0000", 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4980000000000001e-06 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.791 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864566+0000", 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4660000000000001e-06 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864586+0000", 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9403e-05 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864597+0000", 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1782e-05 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.792 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.865289+0000", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000691923 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.865320+0000", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0431000000000001e-05 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898763+0000", 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0334432250000001 2024-09-13T14:43:11.793 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898897+0000", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013364799999999999 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.899526+0000", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00062976899999999997 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924433+0000", 2024-09-13T14:43:11.794 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.024906927999999998 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927694+0000", 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003260256 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927742+0000", 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8282999999999998e-05 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.795 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927815+0000", 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2917000000000007e-05 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927872+0000", 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7747999999999997e-05 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187708 8.3 8:cf4a0971:::benchmark_data_smithi042_130700_object63119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:43:11.796 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.05777921700002, 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.023983487, 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187708 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:43:11.797 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904088+0000", 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2720000000000001e-06 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.798 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904093+0000", 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1350000000000002e-06 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904102+0000", 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6299999999999993e-06 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.799 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904108+0000", 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1820000000000003e-06 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.905077+0000", 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00096839400000000003 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.905088+0000", 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1216000000000001e-05 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.800 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918236+0000", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0131478519999999 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918369+0000", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013295000000000001 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918512+0000", 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014321000000000001 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.801 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927703+0000", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0091908170000000004 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927937+0000", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023419900000000001 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927983+0000", 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5988000000000001e-05 2024-09-13T14:43:11.802 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928034+0000", 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0537000000000001e-05 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928071+0000", 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7104999999999998e-05 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.803 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187719 8.3 8:cec909fa:::benchmark_data_smithi042_130700_object63130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.04553448299998, 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0193972339999999, 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187719 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.804 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916334+0000", 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.818e-06 2024-09-13T14:43:11.805 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916342+0000", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9219999999999997e-06 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916360+0000", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8223000000000002e-05 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916370+0000", 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0088999999999999e-05 2024-09-13T14:43:11.806 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916613+0000", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024308700000000001 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916624+0000", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1252e-05 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921255+0000", 2024-09-13T14:43:11.807 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0046310679999999 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921382+0000", 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012638100000000001 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921523+0000", 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014164099999999999 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.808 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927947+0000", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064236600000000003 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928145+0000", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019809999999999999 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935622+0000", 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0074763360000000001 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.809 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935681+0000", 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9567000000000001e-05 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935729+0000", 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8090000000000002e-05 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.810 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187733 8.3 8:cb6a9569:::benchmark_data_smithi042_130700_object63144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.02969597499998, 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0037710639999999, 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187733 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.811 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932172+0000", 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:43:11.812 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932178+0000", 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2469999999999997e-06 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932193+0000", 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4273999999999999e-05 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.813 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932201+0000", 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6710000000000005e-06 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932268+0000", 2024-09-13T14:43:11.814 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6944000000000001e-05 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932278+0000", 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5559999999999996e-06 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924468+0000", 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99218998800000002 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.815 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924562+0000", 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4061999999999995e-05 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924703+0000", 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014133599999999999 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.816 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928170+0000", 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034668429999999998 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935774+0000", 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076034570000000001 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.817 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935825+0000", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1285000000000001e-05 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935900+0000", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5479e-05 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935942+0000", 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1179e-05 2024-09-13T14:43:11.818 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187739 8.3 8:c5e9808e:::benchmark_data_smithi042_130700_object63150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 303.02050140099999, 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99483940699999995, 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.819 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187739 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.820 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941367+0000", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.702e-06 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941373+0000", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7309999999999996e-06 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941387+0000", 2024-09-13T14:43:11.821 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3618000000000001e-05 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941396+0000", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5590000000000006e-06 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941466+0000", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0840000000000006e-05 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941478+0000", 2024-09-13T14:43:11.822 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1802e-05 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928378+0000", 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98689947099999997 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928464+0000", 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6094000000000006e-05 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:43:11.823 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928566+0000", 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010249 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935785+0000", 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072182069999999999 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935993+0000", 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020831300000000001 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.824 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936117+0000", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012453799999999999 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936169+0000", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1196000000000002e-05 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936204+0000", 2024-09-13T14:43:11.825 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5846000000000002e-05 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225301 8.1 8:816db9f2:::benchmark_data_smithi042_130700_object100712:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 254.15788940300001, 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.055774234, 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.826 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225301 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.827 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803978+0000", 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4339999999999999e-06 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803984+0000", 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8019999999999997e-06 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.828 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803997+0000", 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3149999999999999e-05 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804007+0000", 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2939999999999994e-06 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804075+0000", 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8554999999999997e-05 2024-09-13T14:43:11.829 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op must wait for map", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804083+0000", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6650000000000003e-06 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807396+0000", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003313561 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807405+0000", 2024-09-13T14:43:11.830 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0669999999999992e-06 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.859673+0000", 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0522687049999999 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.859751+0000", 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8001000000000001e-05 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.831 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225319 8.1 8:8a6235bd:::benchmark_data_smithi042_130700_object100730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 254.15670401, 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0854911300000001, 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225319 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.832 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.833 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805163+0000", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0279999999999999e-06 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805168+0000", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3270000000000002e-06 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805178+0000", 2024-09-13T14:43:11.834 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0237e-05 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805185+0000", 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9480000000000002e-06 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805235+0000", 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9752000000000001e-05 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:43:11.835 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805241+0000", 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9279999999999998e-06 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807424+0000", 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021835499999999998 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807430+0000", 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4399999999999996e-06 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.836 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860239+0000", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0528097700000001 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860357+0000", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117201 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860943+0000", 2024-09-13T14:43:11.837 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00058651199999999997 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882766+0000", 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021822478999999999 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882837+0000", 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1075000000000006e-05 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.838 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890439+0000", 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076019579999999998 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890592+0000", 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015348400000000001 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890654+0000", 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1440999999999997e-05 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.839 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225326 8.1 8:8dc278eb:::benchmark_data_smithi042_130700_object100737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 254.14506931700001, 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.07410107, 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225326 2024-09-13T14:43:11.840 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816798+0000", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0699999999999997e-07 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816800+0000", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.176e-06 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816806+0000", 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.421e-06 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.842 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816811+0000", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7639999999999996e-06 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816908+0000", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6739999999999999e-05 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816918+0000", 2024-09-13T14:43:11.843 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0036e-05 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876442+0000", 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.059524414 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876582+0000", 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000139326 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.844 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876725+0000", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000143304 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890664+0000", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013939058000000001 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890706+0000", 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2361999999999999e-05 2024-09-13T14:43:11.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890766+0000", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9821000000000002e-05 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890852+0000", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6096999999999999e-05 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890898+0000", 2024-09-13T14:43:11.846 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6044000000000002e-05 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225356 8.1 8:8e84ccef:::benchmark_data_smithi042_130700_object100767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 254.11225661399999, 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0491585130000001, 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.847 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225356 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.848 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849611+0000", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.378e-06 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849617+0000", 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2989999999999996e-06 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.849 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849628+0000", 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1215e-05 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849635+0000", 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3880000000000002e-06 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.850 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849696+0000", 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1148000000000005e-05 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849706+0000", 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8120000000000002e-06 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879680+0000", 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0299740319999999 2024-09-13T14:43:11.851 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879768+0000", 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7418999999999998e-05 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879844+0000", 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6153000000000001e-05 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.852 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890717+0000", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010873551 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898610+0000", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078932029999999997 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898661+0000", 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0658999999999997e-05 2024-09-13T14:43:11.853 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898717+0000", 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6248000000000001e-05 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898768+0000", 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1007999999999997e-05 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.854 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225369 8.1 8:82518930:::benchmark_data_smithi042_130700_object100780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 254.09359921000001, 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0307719120000001, 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225369 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.855 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868269+0000", 2024-09-13T14:43:11.856 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6029999999999999e-06 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868275+0000", 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7640000000000002e-06 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868298+0000", 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3754e-05 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:11.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868306+0000", 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7260000000000002e-06 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868365+0000", 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8529999999999997e-05 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868374+0000", 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5589999999999994e-06 2024-09-13T14:43:11.858 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882877+0000", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0145028629999999 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882967+0000", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0200999999999999e-05 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.883039+0000", 2024-09-13T14:43:11.859 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1970000000000004e-05 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890722+0000", 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076824600000000003 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898810+0000", 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0080881570000000003 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:43:11.860 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898866+0000", 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5825000000000002e-05 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898993+0000", 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012693399999999999 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.899039+0000", 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6566000000000001e-05 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.861 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.862 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:11.862 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:11.862 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:43:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:11.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:11.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:11.964 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:12.000 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:12.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:12.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:11 smithi042 ceph-mon[108570]: pgmap v516: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:43:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:11 smithi042 ceph-mon[113926]: pgmap v516: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:43:12.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:12.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:12.236 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:11 smithi159 ceph-mon[102519]: pgmap v516: 99 pgs: 99 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:43:12.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:12.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:12.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:12.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:12.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:12.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:12.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:12.575 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 96809 96793 108.012 75.1875 0.0151516 0.0092535 2024-09-13T14:43:12.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:12.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:12.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:12.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:12.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:12.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:12.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:12.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:13.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:43:13.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:13.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:13.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:13.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:13.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:13.404 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:13.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:13.481 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:13.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:13.550 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:13.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:13.576 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 97942 97926 107.358 70.8125 0.0119594 0.00930789 2024-09-13T14:43:13.670 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:13.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:13.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:13.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:13.883 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:13.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:13.985 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:14.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:13 smithi042 ceph-mon[108570]: pgmap v517: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:43:14.013 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:13 smithi042 ceph-mon[113926]: pgmap v517: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:43:14.014 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:14.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:13 smithi159 ceph-mon[102519]: pgmap v517: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:43:14.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:14.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:14.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:14.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:14.397 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.447 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:14.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:14.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:14.576 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 99058 99042 106.71 69.75 0.0178127 0.00936586 2024-09-13T14:43:14.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:14.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:14.724 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:14.728 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:14.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:14.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:14.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:14.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:14.973 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:15.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:15.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:15.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:15.579 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 100148 100132 106.05 68.125 0.0188703 0.00942356 2024-09-13T14:43:15.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:15.704 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.764 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:15.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:15.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:15.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:16.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:16.056 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:16.058 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:15 smithi042 ceph-mon[113926]: pgmap v518: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:16.062 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:15 smithi042 ceph-mon[108570]: pgmap v518: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:16.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:16.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.190 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:16.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:15 smithi159 ceph-mon[102519]: pgmap v518: 99 pgs: 99 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:16.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:16.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:16.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:16.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:16.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:16.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:16.581 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:43:16.581792+0000 min lat: 0.00132914 max lat: 0.38602 avg lat: 0.00951199 2024-09-13T14:43:16.581 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:43:16.582 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 100906 100890 105.069 47.375 0.0131985 0.00951199 2024-09-13T14:43:16.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:16.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:16.700 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.730 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:16.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:16.831 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:16.872 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:16.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:16.973 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:17.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:17.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:17.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:17.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:17.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:17.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.277 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:17.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:17.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:17.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:17.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:17.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:17.583 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 101912 101896 104.374 62.875 0.0171727 0.00957578 2024-09-13T14:43:17.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:17.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:17.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:17.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:17.852 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:17.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:17.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:17.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.001 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:18.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[113926]: pgmap v519: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[108570]: pgmap v519: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:17 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.141 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:18.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:18.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:18.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:17 smithi159 ceph-mon[102519]: pgmap v519: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:18.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:17 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:17 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:18.274 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:18.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:18.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:18.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:18.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:18.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:18.565 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:18.583 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 102832 102816 103.618 57.5 0.0208279 0.00964519 2024-09-13T14:43:18.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:18.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:18.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:18.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:18.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:18.943 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:18.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:18.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:18.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:19.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:19.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:19.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:19.014 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:19.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:19.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:19.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:19.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:43:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:43:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:19.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:19.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:19.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:19.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:19.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:19.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:19.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 103946 103930 103.078 69.625 0.0105665 0.00969641 2024-09-13T14:43:19.640 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:19.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:19.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:19.780 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:19.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:19.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:19.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:19] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-09-13T14:43:19.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:19 smithi042 ceph-mon[113926]: pgmap v520: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:43:19.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:19 smithi042 ceph-mon[108570]: pgmap v520: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:43:19.929 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:19.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:20.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:20.047 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:20.048 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:20.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:20.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:20.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:19 smithi159 ceph-mon[102519]: pgmap v520: 99 pgs: 99 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T14:43:20.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:20.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:20.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:20.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:20.531 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:20.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:20.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 104992 104976 102.489 65.375 0.011886 0.00975155 2024-09-13T14:43:20.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:20.701 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:20.773 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:43:20.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:20.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:20.843 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 9 pgp_num to 16 2024-09-13T14:43:20.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:20.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:20.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:20.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1591451712' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1591451712' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.003 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:21.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.099 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:21.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:21.245 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1591451712' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:43:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:21.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:21.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:21.521 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:21.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 106050 106034 101.93 66.125 0.0153668 0.00980552 2024-09-13T14:43:21.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:21.651 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:21.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:21.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:21.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:21.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:21.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:21.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:21.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:22.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:22.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[113926]: pgmap v521: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:43:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:43:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[113926]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:43:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[108570]: pgmap v521: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:43:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:43:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:21 smithi042 ceph-mon[108570]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:43:22.158 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:22.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:21 smithi159 ceph-mon[102519]: pgmap v521: 99 pgs: 99 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2024-09-13T14:43:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:21 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:43:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:21 smithi159 ceph-mon[102519]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T14:43:22.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:22.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:22.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:22.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:22.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 107124 107108 101.402 67.125 0.0227966 0.00985653 2024-09-13T14:43:22.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:22.623 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:22.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:22.760 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:22.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:22.876 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:43:22.926 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:22.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:22.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:23.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:23.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:23.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:23.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:23.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3466213193' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:22 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3466213193' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:22 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3466213193' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:22 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:23.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:23.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:23.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:23.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:23.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:23.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 108245 108229 100.934 70.0625 0.02482 0.0099024 2024-09-13T14:43:23.612 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:23.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:23.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:23.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:23.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:23.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:23.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:23.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:24.024 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:24.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[108570]: pgmap v523: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[108570]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[113926]: pgmap v523: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:23 smithi042 ceph-mon[113926]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:43:24.126 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:24.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:23 smithi159 ceph-mon[102519]: pgmap v523: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:23 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:23 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:23 smithi159 ceph-mon[102519]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T14:43:24.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:24.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:24.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:24.440 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:24.526 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:24.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:24.586 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 109328 109312 100.445 67.6875 0.0215456 0.00994979 2024-09-13T14:43:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:24.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:24.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:24.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:24.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:24.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:24.880 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:24.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:24.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:24.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:25.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:25.286 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:25.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:25.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:25.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:25.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:25.584 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 110430 110414 99.9875 68.875 0.0156305 0.00999616 2024-09-13T14:43:25.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:25.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:25.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:25.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:25.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:25.858 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:43:25.859 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:25.906 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:43:25.906 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:25.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:25.944 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:43:25.944 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:25.944 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:25 smithi042 ceph-mon[108570]: pgmap v525: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:25.944 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2475276080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:25.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:25 smithi042 ceph-mon[113926]: pgmap v525: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:25.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2475276080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:25.983 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:43:25.983 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:25.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:25.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:26.017 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:43:26.018 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:26.054 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:43:26.054 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:26.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:26.088 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:43:26.089 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:26.121 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:43:26.121 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:26.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:26.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:26.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:25 smithi159 ceph-mon[102519]: pgmap v525: 99 pgs: 99 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:43:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2475276080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:26.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:26.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:26.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:26.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:26.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:26.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:26.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:43:26.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:26.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:26.585 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 111473 111457 99.4904 65.1875 0.0097372 0.0100456 2024-09-13T14:43:26.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:26.656 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:26.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:26.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:26.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:26.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:26.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:26.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:26.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:27.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:27.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:27.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:27.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:27.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:27.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:27.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:27.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:27.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:27.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:27.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:27.585 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 112054 112038 98.6007 36.3125 0.0111209 0.0101368 2024-09-13T14:43:27.680 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:27.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:27.824 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:27.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:27.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:27.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:27.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:28.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:28.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:28.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:27 smithi159 ceph-mon[102519]: pgmap v526: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:28.268 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:27 smithi042 ceph-mon[108570]: pgmap v526: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:28.268 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:27 smithi042 ceph-mon[113926]: pgmap v526: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:28.277 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.339 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:43:28.340 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:43:28.340 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T14:43:28.340 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2024-09-13T14:43:28.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 0 2024-09-13T14:43:28.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:28.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.439 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:28.540 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.585 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 113054 113038 98.0991 62.5 0.015349 0.0101889 2024-09-13T14:43:28.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:28.728 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:28.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:28.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:28.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:28.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:28.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:28.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:29.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:29.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:29.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:29.290 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:29.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:29.439 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:29.541 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:29.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 114131 114115 97.6719 67.3125 0.0227801 0.0102335 2024-09-13T14:43:29.606 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:29.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:29.709 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:29.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.766 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:29.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:29.822 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:29] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-09-13T14:43:29.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:29.868 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:43:29.868 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:43:29.868 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:43:29.868 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:43:29.868 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:43:29.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:29.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:30.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.049 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:30.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:29 smithi042 ceph-mon[113926]: pgmap v527: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:43:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:29 smithi042 ceph-mon[108570]: pgmap v527: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:43:30.150 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:30.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:29 smithi159 ceph-mon[102519]: pgmap v527: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T14:43:30.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:30.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:30.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:30.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:30.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:30.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:30.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:30.589 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 115208 115192 97.2617 67.3125 0.0121185 0.0102769 2024-09-13T14:43:30.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:30.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:30.636 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:30.640 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:30.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:43:30.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:30.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:30.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:30.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:30.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:30.974 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:31.011 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:31.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:31.143 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:31.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:31.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:31.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:31.289 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:43:31.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:31.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:31.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:31.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:31.589 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 116203 116187 96.794 62.1875 0.0177799 0.0103256 2024-09-13T14:43:31.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:31.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:31.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:31.735 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:31.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:31.836 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:31.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:31.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:31.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:32.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.022 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:32.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:31 smithi042 ceph-mon[113926]: pgmap v528: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-13T14:43:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:31 smithi042 ceph-mon[108570]: pgmap v528: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-13T14:43:32.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:32.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:32.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:31 smithi159 ceph-mon[102519]: pgmap v528: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 959 op/s 2024-09-13T14:43:32.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:32.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:32.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:32.411 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:32.449 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:32.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:32.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:32.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:32.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 117298 117282 96.4209 68.4375 0.00627625 0.0103662 2024-09-13T14:43:32.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:32.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:32.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:32.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:32.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:32.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:32.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:33.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:33.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:33.172 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:33.172 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:33.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:33.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:33.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:33.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.480 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:33.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:33.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:33.589 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 118312 118296 95.9917 63.375 0.0120692 0.0104122 2024-09-13T14:43:33.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:33.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:33.717 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:33.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:33.868 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:33.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:33.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:33.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:34.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:34.027 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:33 smithi042 ceph-mon[113926]: pgmap v529: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:34.028 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:33 smithi042 ceph-mon[108570]: pgmap v529: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:34.038 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:34.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:34.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:34.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:34.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:33 smithi159 ceph-mon[102519]: pgmap v529: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:43:34.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:34.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:34.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:34.447 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:34.549 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:34.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:34.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:34.589 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 119226 119210 95.4934 57.125 0.0268394 0.0104674 2024-09-13T14:43:34.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:34.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:34.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:34.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:34.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:34.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:34.901 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:43:34.904 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.7 primary-affinity to 0 (802) 2024-09-13T14:43:34.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.005 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:35.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:35.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.128 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.230 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:35.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3620818251' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:34 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:35.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3620818251' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3620818251' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.0} : dispatch 2024-09-13T14:43:35.255 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:43:35.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:35.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:35.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:35.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:35.501 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:35.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 119645 119629 94.6163 26.1875 0.00382987 0.0104822 2024-09-13T14:43:35.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:35.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:35.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:35.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:35.851 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:35.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:35.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:35.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:36.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:36.128 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:36.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:36.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:36.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: pgmap v530: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:35 smithi159 ceph-mon[102519]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:43:36.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: pgmap v530: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:43:36.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[108570]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: pgmap v530: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 984 op/s 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3690998456' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:36.263 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:35 smithi042 ceph-mon[113926]: osdmap e157: 8 total, 8 up, 8 in 2024-09-13T14:43:36.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:36.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.300 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:36.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:36.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:36.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:36.543 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:36.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:36.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:43:36.590242+0000 min lat: 0.00132914 max lat: 1.02758 avg lat: 0.0106338 2024-09-13T14:43:36.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:43:36.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 120362 120346 93.9938 44.8125 0.00924471 0.0106338 2024-09-13T14:43:36.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:36.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:36.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:36.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:36.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:36.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:37.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:37.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:37.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:36 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:43:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:36 smithi159 ceph-mon[102519]: 1.2 scrub ok 2024-09-13T14:43:37.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:37.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:37.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:36 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:43:37.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:36 smithi042 ceph-mon[113926]: 1.2 scrub ok 2024-09-13T14:43:37.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:36 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:43:37.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:36 smithi042 ceph-mon[108570]: 1.2 scrub ok 2024-09-13T14:43:37.389 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:37.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:37.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:37.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:37.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 121436 121420 93.6621 67.125 0.0133258 0.0106719 2024-09-13T14:43:37.669 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:37.706 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:37.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:37.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:37.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:37.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:37.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:37.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:37.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:37.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:38.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:38.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:38.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[113926]: pgmap v533: 99 pgs: 3 peering, 96 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:43:38.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:43:38.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[113926]: 1.0 scrub starts 2024-09-13T14:43:38.127 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[113926]: 1.0 scrub ok 2024-09-13T14:43:38.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:38.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[108570]: pgmap v533: 99 pgs: 3 peering, 96 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:43:38.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:43:38.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[108570]: 1.0 scrub starts 2024-09-13T14:43:38.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:37 smithi042 ceph-mon[108570]: 1.0 scrub ok 2024-09-13T14:43:38.230 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:37 smithi159 ceph-mon[102519]: pgmap v533: 99 pgs: 3 peering, 96 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-13T14:43:38.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:37 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:43:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:37 smithi159 ceph-mon[102519]: 1.0 scrub starts 2024-09-13T14:43:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:37 smithi159 ceph-mon[102519]: 1.0 scrub ok 2024-09-13T14:43:38.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:38.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:38.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:38.415 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:38.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:38.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:38.551 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:38.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:38.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 122487 122471 93.3209 65.6875 0.0145853 0.0107109 2024-09-13T14:43:38.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:38.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:38.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:38.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:38.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:38.859 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:43:38.859 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:83772 9.6 e152/139 9:698dafff:::benchmark_data_smithi042_157727_object83771:head v 152'5100, mlcod=152'5095)", 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:00.971674+0000", 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 37.884435531000001, 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021284566000000001, 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.860 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971674+0000", 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971674+0000", 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.861 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971676+0000", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1370000000000001e-06 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971725+0000", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9382000000000003e-05 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971737+0000", 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1640999999999999e-05 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.862 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971744+0000", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7710000000000006e-06 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971795+0000", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0460999999999997e-05 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.971814+0000", 2024-09-13T14:43:38.863 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9487000000000002e-05 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.992715+0000", 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020900670999999999 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.864 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.992959+0000", 2024-09-13T14:43:38.865 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000244016 2024-09-13T14:43:38.865 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.865 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.865 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.865 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:83783 9.6 e152/139 9:6655774b:::benchmark_data_smithi042_157727_object83782:head v 152'5101, mlcod=152'5097)", 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:00.974164+0000", 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 37.881945588000001, 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0189648, 2024-09-13T14:43:38.866 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974164+0000", 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.867 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974164+0000", 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974169+0000", 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4050000000000002e-06 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.868 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974215+0000", 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6063000000000003e-05 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974236+0000", 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0951999999999999e-05 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.974245+0000", 2024-09-13T14:43:38.869 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8170000000000005e-06 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.975300+0000", 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010550640000000001 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.975354+0000", 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4724e-05 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.870 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.992962+0000", 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017607767999999999 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:00.993129+0000", 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000167007 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.871 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:89170 9.0 e153/139 9:0c00bcda:::benchmark_data_smithi042_157727_object89169:head v 153'5528, mlcod=153'5526)", 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 33.241609543000003, 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.027526489000000001, 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.872 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614502+0000", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3060000000000001e-06 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614596+0000", 2024-09-13T14:43:38.873 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4013999999999998e-05 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614636+0000", 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0667999999999998e-05 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614650+0000", 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3679999999999999e-05 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.874 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614733+0000", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.276e-05 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614763+0000", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0431000000000001e-05 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624424+0000", 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096609380000000009 2024-09-13T14:43:38.875 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.642027+0000", 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017602692 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:89176 9.6 e153/139 9:656e9a48:::benchmark_data_smithi042_157727_object89175:head v 153'5445, mlcod=153'5442)", 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:05.624722+0000", 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 33.231387906999998, 2024-09-13T14:43:38.876 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017293033999999999, 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624722+0000", 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.877 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624722+0000", 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624723+0000", 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3069999999999999e-06 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624788+0000", 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5090000000000002e-05 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.878 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624805+0000", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6812000000000001e-05 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624817+0000", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2109999999999999e-05 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624953+0000", 2024-09-13T14:43:38.879 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000135823 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624988+0000", 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4353999999999997e-05 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.641905+0000", 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016917881999999999 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.880 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.642015+0000", 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109656 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111577 9.2 e155/139 9:458b1f83:::benchmark_data_smithi042_157727_object111576:head v 155'6951, mlcod=155'6948)", 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 12.091367096999999, 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36778615199999998, 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.881 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.882 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764744+0000", 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0240000000000001e-06 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764793+0000", 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8990999999999999e-05 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.883 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764806+0000", 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2989e-05 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764814+0000", 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3100000000000001e-06 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764896+0000", 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2065999999999997e-05 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.884 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764923+0000", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7109000000000001e-05 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132474+0000", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36755078200000002 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132529+0000", 2024-09-13T14:43:38.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4880999999999999e-05 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111592 9.2 e155/139 9:410d9e3a:::benchmark_data_smithi042_157727_object111591:head v 155'6952, mlcod=155'6949)", 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 12.084761903, 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36121327600000003, 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.886 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771349+0000", 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.353e-06 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771399+0000", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0086999999999997e-05 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119327+0000", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34792794599999999 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119345+0000", 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7586000000000001e-05 2024-09-13T14:43:38.888 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.122692+0000", 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033467879999999998 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.122713+0000", 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0973999999999999e-05 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.889 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132537+0000", 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098238549999999994 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132561+0000", 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4686999999999999e-05 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111590 9.6 e155/139 9:6fbdf5c6:::benchmark_data_smithi042_157727_object111589:head v 155'6837, mlcod=155'6834)", 2024-09-13T14:43:38.890 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 12.084743653, 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.35755826299999999, 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.891 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.892 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771367+0000", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.031e-06 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771408+0000", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0262000000000002e-05 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119343+0000", 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34793492199999998 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.893 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119354+0000", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1268e-05 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120248+0000", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00089436300000000002 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120269+0000", 2024-09-13T14:43:38.894 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0554999999999999e-05 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128883+0000", 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086147319999999999 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128925+0000", 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1130000000000001e-05 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.895 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111608 9.0 e155/139 9:0bc62cae:::benchmark_data_smithi042_157727_object111607:head v 155'6920, mlcod=155'6918)", 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 12.066570096, 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34107008799999999, 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:43:38.896 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789541+0000", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.254e-06 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789594+0000", 2024-09-13T14:43:38.897 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2549999999999997e-05 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119370+0000", 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32977673600000001 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119381+0000", 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0505e-05 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.898 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119981+0000", 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060055600000000003 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120005+0000", 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3292e-05 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.130556+0000", 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010551328 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.130610+0000", 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3866999999999998e-05 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111633 9.2 e155/139 9:4f832cbb:::benchmark_data_smithi042_157727_object111632:head v 155'6955, mlcod=155'6949)", 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:27.119535+0000", 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 11.736575441999999, 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016499653, 2024-09-13T14:43:38.900 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119535+0000", 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119535+0000", 2024-09-13T14:43:38.901 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119535+0000", 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2000000000000001e-07 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119588+0000", 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3254000000000002e-05 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119595+0000", 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9890000000000003e-06 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119601+0000", 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3500000000000002e-06 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128269+0000", 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086680409999999996 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128290+0000", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0958000000000002e-05 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.135893+0000", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076027619999999999 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.136034+0000", 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014097900000000001 2024-09-13T14:43:38.904 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111637 9.2 e155/139 9:4caa1eda:::benchmark_data_smithi042_157727_object111636:head v 155'6956, mlcod=155'6949)", 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 11.736501053, 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022970880999999999, 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.905 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.79e-07 2024-09-13T14:43:38.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119657+0000", 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.795e-05 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119664+0000", 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4420000000000003e-06 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119668+0000", 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3089999999999997e-06 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.129908+0000", 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010240196 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.129928+0000", 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0316999999999999e-05 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.908 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142519+0000", 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012590565 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142580+0000", 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0822999999999997e-05 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.909 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111640 9.2 e155/139 9:44090333:::benchmark_data_smithi042_157727_object111639:head v 155'6957, mlcod=155'6949)", 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 11.736435224999999, 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022948066, 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.910 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119756+0000", 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1242000000000006e-05 2024-09-13T14:43:38.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119763+0000", 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2929999999999997e-06 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119767+0000", 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3200000000000001e-06 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.912 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132590+0000", 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012823413000000001 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132612+0000", 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1537000000000001e-05 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142589+0000", 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0099768800000000005 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142623+0000", 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4047000000000001e-05 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:114542 9.6 e155/139 9:67c829a0:::benchmark_data_smithi042_157727_object114541:head v 155'7013, mlcod=155'7010)", 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:30.005240+0000", 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 8.8508700559999998, 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019229690000000001, 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.914 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005240+0000", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005240+0000", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005241+0000", 2024-09-13T14:43:38.915 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.54e-07 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005270+0000", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9079000000000001e-05 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005289+0000", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8757999999999999e-05 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005296+0000", 2024-09-13T14:43:38.916 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9070000000000007e-06 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005354+0000", 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7386000000000002e-05 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.005380+0000", 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5772999999999999e-05 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.024194+0000", 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018814188999999999 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.024470+0000", 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00027574400000000002 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:114545 9.6 e155/139 9:6b0dc329:::benchmark_data_smithi042_157727_object114544:head v 155'7014, mlcod=155'7010)", 2024-09-13T14:43:38.918 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:30.006673+0000", 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 8.8494372800000001, 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017869315, 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006673+0000", 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.919 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006673+0000", 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006673+0000", 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8299999999999996e-07 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006755+0000", 2024-09-13T14:43:38.921 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1303000000000004e-05 2024-09-13T14:43:38.921 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006768+0000", 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3591000000000001e-05 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.006775+0000", 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4189999999999999e-06 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.922 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.009048+0000", 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022730340000000002 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.009196+0000", 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014833700000000001 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.024477+0000", 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015280686 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.923 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.024542+0000", 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5161999999999997e-05 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:114703 9.6 e155/139 9:6b7926c0:::benchmark_data_smithi042_157727_object114702:head v 155'7027, mlcod=155'7024)", 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:30.145154+0000", 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 8.7109555669999992, 2024-09-13T14:43:38.924 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016766352000000002, 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145154+0000", 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145154+0000", 2024-09-13T14:43:38.925 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145155+0000", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7899999999999997e-07 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145192+0000", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6361e-05 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145211+0000", 2024-09-13T14:43:38.926 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9740000000000001e-05 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.145220+0000", 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9120000000000004e-06 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.146298+0000", 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001077684 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.146355+0000", 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7009999999999998e-05 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.161865+0000", 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015509657 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:30.161921+0000", 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6109000000000001e-05 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.928 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:116081 9.2 e155/139 9:4cd2ce73:::benchmark_data_smithi042_157727_object116080:head v 155'7196, mlcod=155'7194)", 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:31.471589+0000", 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 7.3845211989999999, 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020278600000000001, 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.929 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471589+0000", 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471589+0000", 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471590+0000", 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5899999999999995e-07 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.930 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471614+0000", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4017999999999999e-05 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471633+0000", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9225999999999999e-05 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471640+0000", 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0049999999999998e-06 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471698+0000", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8678999999999998e-05 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.471719+0000", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0601000000000001e-05 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.491547+0000", 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019828143999999999 2024-09-13T14:43:38.932 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:31.491867+0000", 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00032016800000000002 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:117253 9.6 e155/139 9:64f620dd:::benchmark_data_smithi042_157727_object117252:head v 155'7181, mlcod=155'7179)", 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:32.545790+0000", 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 6.3103196199999996, 2024-09-13T14:43:38.933 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017053768, 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545790+0000", 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545790+0000", 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.934 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545791+0000", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1399999999999995e-07 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545834+0000", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2895999999999997e-05 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545851+0000", 2024-09-13T14:43:38.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7010000000000001e-05 2024-09-13T14:43:38.936 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.936 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.936 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.545866+0000", 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5e-05 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.546041+0000", 2024-09-13T14:43:38.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017463999999999999 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.546089+0000", 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8446999999999999e-05 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.562759+0000", 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016669722000000001 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:32.562844+0000", 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5138999999999999e-05 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:118264 9.6 e155/139 9:6fe26711:::benchmark_data_smithi042_157727_object118263:head v 155'7240, mlcod=155'7237)", 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:33.548706+0000", 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 5.3074034799999996, 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018265969, 2024-09-13T14:43:38.939 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548706+0000", 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548706+0000", 2024-09-13T14:43:38.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548708+0000", 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.356e-06 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548750+0000", 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2255000000000001e-05 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548772+0000", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1518e-05 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548781+0000", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3540000000000006e-06 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548852+0000", 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0761999999999998e-05 2024-09-13T14:43:38.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.548896+0000", 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4032999999999997e-05 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.566921+0000", 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018025076000000001 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:33.566972+0000", 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1615e-05 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:120214 9.0 e157/139 9:01555bb4:::benchmark_data_smithi042_157727_object120213:head v 157'7467, mlcod=157'7465)", 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:36.457536+0000", 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 2.3985743400000001, 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021473394999999999, 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.944 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457536+0000", 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457536+0000", 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457537+0000", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.677e-06 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457571+0000", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3624999999999997e-05 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457591+0000", 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0591e-05 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457602+0000", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0287e-05 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457665+0000", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2848999999999994e-05 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.457688+0000", 2024-09-13T14:43:38.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3428e-05 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.478956+0000", 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021267793 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.479009+0000", 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3145e-05 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.948 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:120218 9.0 e157/139 9:0b040e67:::benchmark_data_smithi042_157727_object120217:head v 157'7468, mlcod=157'7465)", 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:36.459461+0000", 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 2.3966487839999999, 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019574218000000001, 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459461+0000", 2024-09-13T14:43:38.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459461+0000", 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459462+0000", 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0500000000000002e-07 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.950 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459531+0000", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9005999999999996e-05 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459542+0000", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1059e-05 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.459554+0000", 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1712000000000001e-05 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.951 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.468948+0000", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0093938570000000002 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.468996+0000", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8168999999999999e-05 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.479013+0000", 2024-09-13T14:43:38.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010016969000000001 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:36.479035+0000", 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2640999999999999e-05 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:120960 9.2 e157/139 9:4d9ce210:::benchmark_data_smithi042_157727_object120959:head v 157'7483, mlcod=157'7479)", 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:37.155933+0000", 2024-09-13T14:43:38.953 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.7001765629999999, 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018558075, 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.155933+0000", 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.155933+0000", 2024-09-13T14:43:38.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.155934+0000", 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.037e-06 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.155981+0000", 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7058000000000002e-05 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:43:38.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.155997+0000", 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5591e-05 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.156005+0000", 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5039999999999996e-06 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.164174+0000", 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081696640000000001 2024-09-13T14:43:38.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.164194+0000", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0038e-05 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.174396+0000", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010201729 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:37.174491+0000", 2024-09-13T14:43:38.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5453999999999998e-05 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:43:38.958 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:43:38.998 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:38.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:38.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:39.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:39.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:39.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:43:39.136 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:39.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:43:39.181 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:43:39.181 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:39.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:43:39.216 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:39.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.262 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:43:39.262 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:39.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:39.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.295 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:43:39.295 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:39.335 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:43:39.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:39.370 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:43:39.370 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:39.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.402 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:43:39.402 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:39.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:39.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:39.568 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:39.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:39.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 123352 123336 92.8479 54.0625 0.00926849 0.0107595 2024-09-13T14:43:39.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:39.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.701 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:39.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:39.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:39.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:39.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:39.986 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:39 smithi042 ceph-mon[113926]: pgmap v534: 99 pgs: 3 peering, 96 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4294891092' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:39] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-09-13T14:43:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:39 smithi042 ceph-mon[108570]: pgmap v534: 99 pgs: 3 peering, 96 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4294891092' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:40.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:40.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:40.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:39 smithi159 ceph-mon[102519]: pgmap v534: 99 pgs: 3 peering, 96 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:43:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4294891092' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:40.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:40.309 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:40.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:40.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:40.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:40.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:40.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 124393 124377 92.5172 65.0625 0.0177198 0.0108038 2024-09-13T14:43:40.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:40.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:40.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:40.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:43:40.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:40.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:40.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:40.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:41.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:41.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:41.155 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:41.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:41.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:41.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:41.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:41.522 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:41.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:41.590 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 125380 125364 92.1543 61.6875 0.00975278 0.0108454 2024-09-13T14:43:41.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:41.706 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:41.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:41.762 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:41.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:41.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:41.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:41.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:41.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:42.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:42.139 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:41 smithi042 ceph-mon[108570]: pgmap v535: 99 pgs: 3 peering, 96 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s 2024-09-13T14:43:42.139 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:41 smithi042 ceph-mon[113926]: pgmap v535: 99 pgs: 3 peering, 96 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s 2024-09-13T14:43:42.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:41 smithi159 ceph-mon[102519]: pgmap v535: 99 pgs: 3 peering, 96 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 46 MiB/s wr, 742 op/s 2024-09-13T14:43:42.255 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:42.357 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:42.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:42.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:42.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:42.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:42.591 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 126366 126350 91.7993 61.625 0.0202338 0.0108872 2024-09-13T14:43:42.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:42.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:42.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:42.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:42.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:42.915 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:43:42.915 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:43:42.915 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-09-13T14:43:42.916 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:43:42.916 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:43:42.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-09-13T14:43:42.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:42.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:42.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:43.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:43.074 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:43.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:43.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:43.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:43.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:42 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:43:43.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:42 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:43:43.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:42 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T14:43:43.365 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:43.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:43.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:43.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:43.592 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 127387 127371 91.4759 63.8125 0.00847223 0.0109268 2024-09-13T14:43:43.609 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:43:43.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:43.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:43.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:43.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.807 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:43.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:43.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:43.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:43.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:44.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:44.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:44.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:43 smithi159 ceph-mon[102519]: pgmap v536: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:43:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:44.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:43 smithi042 ceph-mon[108570]: pgmap v536: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:43:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:43 smithi042 ceph-mon[113926]: pgmap v536: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2024-09-13T14:43:44.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:44.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:44.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:44.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:44.592 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 128372 128356 91.1359 61.5625 0.0143766 0.0109675 2024-09-13T14:43:44.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:44.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:44.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:44.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:44.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:44.927 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:44.939 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:44.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:45.041 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:45.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:45.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:45.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:45.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:45.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:45.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:45.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:45.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:45.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:45.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 129224 129208 90.7103 53.25 0.0134981 0.011019 2024-09-13T14:43:45.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:45.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:45.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:45.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:45.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:45.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:45.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:45.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:45.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:45.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:46.028 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:46.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:46.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:46.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:46.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:46.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[113926]: pgmap v537: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2024-09-13T14:43:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3031253305' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[108570]: pgmap v537: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2024-09-13T14:43:46.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3031253305' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:46 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.400 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:43:46.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:46.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:46.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:46 smithi159 ceph-mon[102519]: pgmap v537: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2024-09-13T14:43:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:46 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3031253305' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:46 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:43:46.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:46.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:46.592 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0073 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 129946 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 90.233 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 40.1254 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 218.688 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 26.1875 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1443 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 642.006 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3499 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 419 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.011078 2024-09-13T14:43:46.593 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0144253 2024-09-13T14:43:46.594 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 1.02758 2024-09-13T14:43:46.594 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00132914 2024-09-13T14:43:46.597 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:43:46.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:46.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:46.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:46.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:46.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.859 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:46.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:46.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:46.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:46.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:46.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:47.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.226 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:47.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[113926]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:47 smithi042 ceph-mon[108570]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:43:47.366 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:47 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:43:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:47 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:43:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:47 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:43:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:47 smithi159 ceph-mon[102519]: osdmap e158: 8 total, 8 up, 8 in 2024-09-13T14:43:47.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:47.521 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:47.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:47.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:47.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:47.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:47.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:47.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:47.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:47.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:48.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:48.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:48.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:48.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:48.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:48.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:48.347 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:48 smithi042 ceph-mon[113926]: pgmap v539: 99 pgs: 99 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-09-13T14:43:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:48 smithi042 ceph-mon[108570]: pgmap v539: 99 pgs: 99 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-09-13T14:43:48.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:48.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:48.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:48 smithi159 ceph-mon[102519]: pgmap v539: 99 pgs: 99 active+clean; 7.8 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s 2024-09-13T14:43:48.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:48.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:48.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:48.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:48.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:48.911 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:48.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:49.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:49.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:49.153 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:49.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:49.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:49.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:49.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.449 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.551 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.591 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:49.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:49.610 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:43:49.610 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:43:49.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.826 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.850 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:49.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:49.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:49.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:49.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:49.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:50.025 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:49] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-09-13T14:43:50.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:50.126 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:50.228 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:50.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:50.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:50 smithi042 ceph-mon[108570]: pgmap v540: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 58 MiB/s wr, 949 op/s 2024-09-13T14:43:50.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:50 smithi042 ceph-mon[113926]: pgmap v540: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 58 MiB/s wr, 949 op/s 2024-09-13T14:43:50.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:50.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:50.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:50 smithi159 ceph-mon[102519]: pgmap v540: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 58 MiB/s wr, 949 op/s 2024-09-13T14:43:50.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:50.539 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:50.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:50.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:50.675 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:50.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:50.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:50.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:50.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:50.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:50.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:50.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:51.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:51.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:51.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:51.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:43:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:51.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:51.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:51.439 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:51.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:51.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:51.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.604 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:51.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:51.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:51.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:51.872 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:51.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:51.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:52.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[113926]: pgmap v541: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 955 op/s 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[108570]: pgmap v541: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 955 op/s 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:52.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:52.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.155 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:52.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:51 smithi159 ceph-mon[102519]: pgmap v541: 99 pgs: 99 active+clean; 7.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 955 op/s 2024-09-13T14:43:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:43:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:43:52.256 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:52.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:52.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:52.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:52.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:52.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:52.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:52.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:52.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:52.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:52.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:43:52.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:52.846 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:52.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:52.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:53.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:53.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:53.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:53.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:53.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:53.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:53.400 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:53.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:53.459 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:53.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:53.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:53.672 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.710 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:53.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:53.784 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:43:53.810 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:43:53.810 INFO:teuthology.orchestra.run.smithi042.stdout: "success": "" 2024-09-13T14:43:53.810 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:43:53.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:53.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:53.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:53.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:54.012 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:53 smithi042 ceph-mon[113926]: pgmap v542: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 948 op/s 2024-09-13T14:43:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:53 smithi042 ceph-mon[108570]: pgmap v542: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 948 op/s 2024-09-13T14:43:54.114 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:54.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:53 smithi159 ceph-mon[102519]: pgmap v542: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 40 MiB/s wr, 948 op/s 2024-09-13T14:43:54.256 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:54.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:54.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:54.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:54.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:54.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:54.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:54.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.865 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:54.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:54.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:54.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:54.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:55.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:55.126 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:55.228 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.267 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:55.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:55.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:55.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:55.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:55.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:55.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:55.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:55.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:55.788 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:55.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:55.927 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:55.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:55.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:55.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:56.063 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:43:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:55 smithi042 ceph-mon[108570]: pgmap v543: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 959 op/s 2024-09-13T14:43:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:55 smithi042 ceph-mon[113926]: pgmap v543: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 959 op/s 2024-09-13T14:43:56.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:43:56.205 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:56.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:55 smithi159 ceph-mon[102519]: pgmap v543: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 29 MiB/s wr, 959 op/s 2024-09-13T14:43:56.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:56.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:56.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:56.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:56.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:56.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:56.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:56.792 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:56.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:56.934 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:56.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:56.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.040 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:57.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:57.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:57.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:57.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:57.475 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:43:57.475 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:43:57.491 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:57.510 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:43:57.510 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:43:57.551 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:43:57.551 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:43:57.583 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:43:57.583 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:43:57.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:57.616 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:43:57.616 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:43:57.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:57.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:57.660 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:43:57.660 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:43:57.695 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:43:57.695 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:43:57.731 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:43:57.731 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:43:57.734 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:57.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:57.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:57.895 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:57.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:57.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:58.007 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:57 smithi042 ceph-mon[108570]: pgmap v544: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 11 MiB/s wr, 957 op/s 2024-09-13T14:43:58.008 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:57 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3843834364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:58.008 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:57 smithi042 ceph-mon[113926]: pgmap v544: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 11 MiB/s wr, 957 op/s 2024-09-13T14:43:58.008 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:57 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3843834364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:58.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:43:58.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:43:58.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:58.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:43:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:57 smithi159 ceph-mon[102519]: pgmap v544: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 11 MiB/s wr, 957 op/s 2024-09-13T14:43:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:57 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3843834364' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:43:58.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:58.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:58.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:58.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:58.451 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:58.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:58.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:58.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:58.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:58.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:58.865 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:58.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:58.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:58.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:43:59.002 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:43:59.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:43:59.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:59.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:43:59.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:59.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:59.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:43:59.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:43:59.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:59.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:59.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:43:59.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:43:59.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:43:59.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:43:59.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:43:59.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:43:59 smithi042 ceph-mon[108570]: pgmap v545: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 9.6 MiB/s wr, 919 op/s 2024-09-13T14:43:59.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:43:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:43:59] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-09-13T14:43:59.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:43:59 smithi042 ceph-mon[113926]: pgmap v545: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 9.6 MiB/s wr, 919 op/s 2024-09-13T14:43:59.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:43:59.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:43:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:43:59.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.099 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:00.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:00.226 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:43:59 smithi159 ceph-mon[102519]: pgmap v545: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 9.6 MiB/s wr, 919 op/s 2024-09-13T14:44:00.319 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:44:00.319 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:44:00.319 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:44:00.319 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:44:00.320 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:44:00.328 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:00.366 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:00.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:00.468 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:00.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.617 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:00.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:00.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:00.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:00.798 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:00.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:00.809 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:44:00.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:00.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:00.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:00.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:00.977 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/398678650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.977 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.978 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/398678650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:00.978 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:01.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:01.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:01.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:01.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:44:01.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:01.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/398678650' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:01.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:01.367 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:01.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:01.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:01.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:01.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:01.699 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:44:01.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:44:01.743 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.797 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:01.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:01.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:01.925 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:01.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:01.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:01.992 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:02.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:02.082 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[108570]: pgmap v546: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 906 op/s 2024-09-13T14:44:02.082 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:02.082 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:02.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[108570]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:44:02.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[113926]: pgmap v546: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 906 op/s 2024-09-13T14:44:02.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:02.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:02.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:01 smithi042 ceph-mon[113926]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:44:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.107 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:02.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:02.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:02.239 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:02.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:01 smithi159 ceph-mon[102519]: pgmap v546: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 906 op/s 2024-09-13T14:44:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:01 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:01 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:01 smithi159 ceph-mon[102519]: osdmap e159: 8 total, 8 up, 8 in 2024-09-13T14:44:02.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:02.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:02.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:02.533 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:02.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:02.634 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:02.675 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:02.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:02.818 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:02.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:02.959 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:02.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:02.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:03.061 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:03.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:03.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:03.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:03.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:03.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:03.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:03.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:03.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:03.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:03.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:03.754 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:44:03.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:03.834 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:03 smithi042 ceph-mon[108570]: pgmap v548: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:03.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:03 smithi042 ceph-mon[113926]: pgmap v548: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:03.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:03.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:03.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:04.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:04.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:04.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:04.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:03 smithi159 ceph-mon[102519]: pgmap v548: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:04.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:04.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:04.426 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:04.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.488 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:04.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:04.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:04.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.639 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:04.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:04.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:04.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:04.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:04.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:04.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:04.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:05.029 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:05.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:05.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:05.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.214 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:05.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:05.314 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:44:05.314 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:05.315 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:05.351 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:05.353 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:44:05.353 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:05.387 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:44:05.387 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:05.428 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:44:05.428 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:05.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:05.462 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:44:05.463 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:05.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:05.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:05.507 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:44:05.507 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:05.547 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:44:05.548 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:05.580 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:44:05.580 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:05.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:05.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:05.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:05.738 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:05.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[113926]: pgmap v549: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2090831123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[108570]: pgmap v549: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:05.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2090831123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:05.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:05.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:05.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:06.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:06.139 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:06.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:06.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:05 smithi159 ceph-mon[102519]: pgmap v549: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:44:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:05 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2090831123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:06.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:06.344 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:06.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:06.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:06.614 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:06.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:06.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:06.906 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:06.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:07.008 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:07.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:07.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:07.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:07.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:07.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:07.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:07.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:07.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:07.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:07.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:07.708 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:07.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:07.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:07.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:07.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:07.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:07.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:07.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:08.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:07 smithi042 ceph-mon[113926]: pgmap v550: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:44:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:07 smithi042 ceph-mon[108570]: pgmap v550: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:44:08.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:08.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:08.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:07 smithi159 ceph-mon[102519]: pgmap v550: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:44:08.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:08.279 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:08.280 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:08.306 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:08.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:08.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:08.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:08.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:08.584 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:08.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:08.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:08.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:08.835 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:44:08.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:08.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:08.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:08.969 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:09.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3190314540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:08 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3190314540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:08 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:09.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:09.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3190314540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:08 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:44:09.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.433 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:09.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:09.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:09.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:09.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:09.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:09.825 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:09] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-09-13T14:44:09.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:09.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:09.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:09.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:10.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[113926]: pgmap v551: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:44:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:44:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[113926]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:44:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[108570]: pgmap v551: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:44:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:44:10.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:09 smithi042 ceph-mon[108570]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:44:10.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:10.133 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:10.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:10.235 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:09 smithi159 ceph-mon[102519]: pgmap v551: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:44:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:44:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:09 smithi159 ceph-mon[102519]: osdmap e160: 8 total, 8 up, 8 in 2024-09-13T14:44:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.281 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:10.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:10.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:44:10.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:10.447 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:10.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:10.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:10.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:10.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:10.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:10.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:10.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:10.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:44:10.819 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:10.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:10.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:10.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:10.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:10.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:11.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:11.132 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:11.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:11.200 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:11.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:11.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:11.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:11.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:11.483 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:11.585 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:11.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.622 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:11.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:11.723 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:11.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:11.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:11.869 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:11.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:11.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:11.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:11 smithi159 ceph-mon[102519]: pgmap v553: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:44:12.013 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:11 smithi042 ceph-mon[113926]: pgmap v553: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:44:12.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:11 smithi042 ceph-mon[108570]: pgmap v553: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-09-13T14:44:12.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:12.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:12.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:12.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:12.281 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:12.307 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:12.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:12.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:12.450 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:12.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:12.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:12.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:12.736 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.769 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:12.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:12.848 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:44:12.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:12.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:12.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:12.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:13.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3143443168' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:12 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3143443168' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:12 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3143443168' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.140 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:12 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:13.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:13.303 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:13.303 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:13.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:13.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:13.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:13.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:13.606 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:13.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:13.708 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:13.741 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:13.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.893 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:13.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:13.995 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:13.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:13 smithi159 ceph-mon[102519]: pgmap v554: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:44:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:13 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:13 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:13 smithi159 ceph-mon[102519]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:44:14.033 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:14.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[113926]: pgmap v554: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[113926]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[108570]: pgmap v554: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 992 op/s 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:13 smithi042 ceph-mon[108570]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T14:44:14.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.265 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:14.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.395 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:14.423 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:14.430 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:14.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:14.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:14.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:14.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:14.728 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:14.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:14.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:14.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:14.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:14.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:14.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:14.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:15.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.132 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:15.234 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.281 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:15.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:15.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:15.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:15.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:15.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:15.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:15.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:15.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:15.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:15.771 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:44:15.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:15.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:15.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:15.952 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:16.002 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:16.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:15 smithi042 ceph-mon[113926]: pgmap v556: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-13T14:44:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:15 smithi042 ceph-mon[108570]: pgmap v556: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-13T14:44:16.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:16.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:16.177 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:15 smithi159 ceph-mon[102519]: pgmap v556: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 895 op/s 2024-09-13T14:44:16.244 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:16.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:16.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:16.505 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:16.505 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:16.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:16.574 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:16.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:16.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:16.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:16.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:16.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:16.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:16.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:16.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:16.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:16.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:17.089 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:17.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:17.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:17.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:17.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:44:17.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.284 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:17.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:17.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:17.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:17.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:17.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:17.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:17.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:17.669 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:17.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:17.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:17.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:17.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:17.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:17.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:17.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:17.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:18.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:18.132 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:18.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:18.142 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:44:18.142 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[108570]: pgmap v557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:44:18.143 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2866024263' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.143 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.143 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[113926]: pgmap v557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:44:18.143 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2866024263' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.143 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:17 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.188 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:17 smithi159 ceph-mon[102519]: pgmap v557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-13T14:44:18.188 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:17 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2866024263' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.188 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:17 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:44:18.235 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:18.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:18.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:18.362 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:18.396 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:18.498 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:18.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:18.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:18.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:18.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:18.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:18.767 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:18.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:18.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:18.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:18.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:18.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:18.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:18 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:44:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:18 smithi159 ceph-mon[102519]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:44:18.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:19.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:19.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:19.219 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:19.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[108570]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[113926]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T14:44:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:44:19.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:19.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:19.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:19.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:19.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:19.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:19.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:19.744 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:19.744 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:19.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:19.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:19.911 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:19.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:19.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:20.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:20.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:19 smithi042 ceph-mon[113926]: pgmap v559: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:44:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:19 smithi042 ceph-mon[108570]: pgmap v559: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:44:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:19] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-09-13T14:44:20.146 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:19 smithi159 ceph-mon[102519]: pgmap v559: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-09-13T14:44:20.154 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:20.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:20.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.441 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:20.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:20.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:44:20.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:20.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:20.755 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:20.857 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:20.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:20.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:20.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:20.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:21.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.047 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:21.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:21.148 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:21.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:21.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:21.343 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:21.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:21.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:21.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:21.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:21.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:21.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:21.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:21.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:21.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:21.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:21.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:21.952 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:21.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:21.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:21.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:22.061 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:21 smithi042 ceph-mon[113926]: pgmap v560: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:44:22.061 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:21 smithi042 ceph-mon[108570]: pgmap v560: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:44:22.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:22.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:22.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:21 smithi159 ceph-mon[102519]: pgmap v560: 99 pgs: 99 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:44:22.266 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:22.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:22.282 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:44:22.282 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:22.283 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:22.284 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:22.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:22.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:22.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:22.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:22.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:22.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:22.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:22.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:22.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:22.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:22.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:22.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:22.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:44:23.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:23.083 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:23.160 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/344427982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:23.163 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/344427982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:23.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:23.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/344427982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:23.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:23.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:23.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:23.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:23.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.495 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:23.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:23.503 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:23.596 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:23.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:23.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:23.771 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:23.873 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:23.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:23.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:24.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:23 smithi042 ceph-mon[108570]: pgmap v561: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:44:24.023 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:23 smithi042 ceph-mon[113926]: pgmap v561: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:44:24.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:24.077 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:24.172 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:23 smithi159 ceph-mon[102519]: pgmap v561: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-13T14:44:24.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:24.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:24.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:24.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:24.360 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:24.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.646 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:24.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:24.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:24.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:24.888 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:24.927 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:24.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:25.029 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:25.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:25.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:25.211 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:25.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:25.313 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:25.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:25.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:25.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:25.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.506 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:25.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:25.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:25.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.658 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:25.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:25.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:25.770 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:25.770 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:44:25.770 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:167992 9.b e162/139 9:d4a9bdae:::benchmark_data_smithi042_157727_object38043:head v 162'10568, mlcod=162'10566)", 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.754116+0000", 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.010612954000000001, 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.01065, 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754116+0000", 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.771 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754116+0000", 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754115+0000", 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754121+0000", 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1709999999999999e-06 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754134+0000", 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3045e-05 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754140+0000", 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6690000000000003e-06 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.773 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754193+0000", 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3545999999999999e-05 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.754212+0000", 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9069e-05 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.774 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:167994 9.9 e162/139 9:9396f3db:::benchmark_data_smithi042_157727_object38045:head v 162'10455, mlcod=162'10453)", 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.755887+0000", 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0088416940000000006, 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.008942139, 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755887+0000", 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.775 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755887+0000", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755886+0000", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755894+0000", 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9619999999999993e-06 2024-09-13T14:44:25.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755910+0000", 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6041000000000001e-05 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755917+0000", 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1409999999999999e-06 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.777 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756170+0000", 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00025297299999999998 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756189+0000", 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9111999999999999e-05 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167999 9.6 9:6e98102e:::benchmark_data_smithi042_157727_object38050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.778 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.755940+0000", 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0087888280000000003, 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0089484520000000008, 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167999 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.779 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755940+0000", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755940+0000", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755938+0000", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.780 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755943+0000", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1030000000000001e-06 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755952+0000", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8959999999999992e-06 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755958+0000", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0009999999999998e-06 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.781 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756017+0000", 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8992e-05 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756143+0000", 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000126186 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.782 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756265+0000", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012156599999999999 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.764302+0000", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0080370119999999996 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.764330+0000", 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8342000000000001e-05 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.783 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.34797.0:167980 9.a e162/139)", 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.758010+0000", 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0067179809999999996, 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0069543729999999998, 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758010+0000", 2024-09-13T14:44:25.784 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758010+0000", 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758009+0000", 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758016+0000", 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7909999999999999e-06 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758029+0000", 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.243e-05 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758036+0000", 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5089999999999996e-06 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:167998 9.d e162/156 9:b57f7e36:::benchmark_data_smithi042_157727_object38049:head v 162'10461, mlcod=162'10459)", 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.758395+0000", 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0063336269999999997, 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066182639999999996, 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758395+0000", 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.787 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758395+0000", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758394+0000", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758400+0000", 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1460000000000001e-06 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.788 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758408+0000", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8639999999999996e-06 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758413+0000", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2190000000000003e-06 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758464+0000", 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0992000000000002e-05 2024-09-13T14:44:25.789 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.758485+0000", 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0098e-05 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.34797.0:167995 9.f e162/139)", 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.759823+0000", 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0049055269999999998, 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0052408669999999997, 2024-09-13T14:44:25.790 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759823+0000", 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759823+0000", 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759822+0000", 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759827+0000", 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5549999999999996e-06 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.792 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759837+0000", 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0655e-05 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.759844+0000", 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5980000000000002e-06 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:168001 9.f 9.c94be9af (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.760599+0000", 2024-09-13T14:44:25.793 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0041292669999999998, 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045095880000000001, 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168001 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760599+0000", 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.794 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760599+0000", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760598+0000", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760603+0000", 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.609e-06 2024-09-13T14:44:25.795 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760615+0000", 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1977999999999999e-05 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760622+0000", 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7510000000000002e-06 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.796 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.34797.0:167995 9.f e162/139)", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.760637+0000", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0040909910000000004, 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045179620000000004, 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760637+0000", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760637+0000", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760637+0000", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760640+0000", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7640000000000001e-06 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760646+0000", 2024-09-13T14:44:25.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2179999999999996e-06 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760650+0000", 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2169999999999996e-06 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:168003 9.8 e162/139 9:138d958a:::benchmark_data_smithi042_157727_object38054:head v 162'10531, mlcod=162'10529)", 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.761499+0000", 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0032296859999999998, 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003703647, 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:25.800 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.761499+0000", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.761499+0000", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.761498+0000", 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:25.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.761505+0000", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4569999999999999e-06 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.761531+0000", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5284999999999999e-05 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.762029+0000", 2024-09-13T14:44:25.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00049792 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.762073+0000", 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4601999999999997e-05 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.762092+0000", 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9151999999999999e-05 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.803 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167980 9.a 9:5fb23343:::benchmark_data_smithi042_157727_object38031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.738276+0000", 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.026452111, 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026982038999999999, 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167980 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.804 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738276+0000", 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738276+0000", 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738275+0000", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738284+0000", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4960000000000002e-06 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738294+0000", 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0012e-05 2024-09-13T14:44:25.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738300+0000", 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0090000000000004e-06 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738355+0000", 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5191000000000001e-05 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738489+0000", 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000133177 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738617+0000", 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012878600000000001 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755454+0000", 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.016837053000000001 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.808 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167981 9.a 9:5f8629ee:::benchmark_data_smithi042_157727_object38032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.738314+0000", 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.026414934000000001, 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027016821, 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167981 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.809 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738314+0000", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738314+0000", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738313+0000", 2024-09-13T14:44:25.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738316+0000", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0620000000000001e-06 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738323+0000", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5780000000000004e-06 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.738327+0000", 2024-09-13T14:44:25.811 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.746e-06 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745445+0000", 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071178989999999996 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745574+0000", 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000128219 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745710+0000", 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013687899999999999 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167986 9.f 9:f5b6b549:::benchmark_data_smithi042_157727_object38037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.745289+0000", 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.019439242999999998, 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.020108596999999999, 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.813 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167986 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745289+0000", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745289+0000", 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745288+0000", 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745294+0000", 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3679999999999998e-06 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745310+0000", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5597999999999999e-05 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.745317+0000", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8979999999999999e-06 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746114+0000", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00079680599999999995 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746218+0000", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010445700000000001 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746319+0000", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010073099999999999 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.751990+0000", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056710550000000004 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:25.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752501+0000", 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00051105499999999995 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760259+0000", 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0077585900000000001 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760302+0000", 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2654999999999997e-05 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.818 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167989 9.a 9:5bc5b47f:::benchmark_data_smithi042_157727_object38040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.746195+0000", 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.018532990999999999, 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019290916000000002, 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167989 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746195+0000", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746195+0000", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746194+0000", 2024-09-13T14:44:25.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746200+0000", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2360000000000004e-06 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746210+0000", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0732e-05 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.746217+0000", 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9589999999999998e-06 2024-09-13T14:44:25.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.750960+0000", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047429600000000001 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.751062+0000", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010175699999999999 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.751166+0000", 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103434 2024-09-13T14:44:25.822 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167995 9.f 9:f7853885:::benchmark_data_smithi042_157727_object38046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.752708+0000", 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.012020659, 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012862602000000001, 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.823 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167995 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752708+0000", 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752708+0000", 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752706+0000", 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752712+0000", 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9240000000000004e-06 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752730+0000", 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7411e-05 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752738+0000", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1110000000000008e-06 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752787+0000", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8851999999999999e-05 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.752961+0000", 2024-09-13T14:44:25.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000174277 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753159+0000", 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000198474 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167996 9.7 9.beb63647 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.753068+0000", 2024-09-13T14:44:25.827 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.011660189, 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012598412999999999, 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167996 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753068+0000", 2024-09-13T14:44:25.828 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753068+0000", 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753067+0000", 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753072+0000", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9969999999999998e-06 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753233+0000", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016091 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753244+0000", 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0781000000000001e-05 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.830 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.34797.0:167986 9.f e162/139 ondisk, result = 0)", 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.753263+0000", 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.011465411, 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012457273, 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753263+0000", 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.831 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753263+0000", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753263+0000", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753267+0000", 2024-09-13T14:44:25.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0729999999999998e-06 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753274+0000", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4649999999999997e-06 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753279+0000", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6020000000000002e-06 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760246+0000", 2024-09-13T14:44:25.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0069676510000000001 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.760253+0000", 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0419999999999997e-06 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:167997 9.a 9:5adfb899:::benchmark_data_smithi042_157727_object38048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:25.753314+0000", 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.011414334999999999, 2024-09-13T14:44:25.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012470495999999999, 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 167997 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753314+0000", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:25.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753314+0000", 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753314+0000", 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753317+0000", 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.579e-06 2024-09-13T14:44:25.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753325+0000", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8560000000000007e-06 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.753330+0000", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3800000000000002e-06 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755640+0000", 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023093800000000002 2024-09-13T14:44:25.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.755915+0000", 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00027523400000000001 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:25.756031+0000", 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000115969 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:44:25.838 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 17 2024-09-13T14:44:25.839 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:25.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:25.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:25.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:25.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:25.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:26.080 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:26.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:26.192 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:25 smithi159 ceph-mon[102519]: pgmap v562: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:44:26.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:26.267 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:26.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:26.300 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:25 smithi042 ceph-mon[108570]: pgmap v562: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:44:26.300 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:25 smithi042 ceph-mon[113926]: pgmap v562: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:44:26.369 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:26.404 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:26.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:26.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:26.537 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:26.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:26.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:26.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:26.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:44:26.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:26.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:26.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:26.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:26.954 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:27.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:27.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:27.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:27.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:27.253 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:27.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:27.395 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:27.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:27.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:27.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:27.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:27.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:27.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.743 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:27.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:27.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:27.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:27.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:27.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:28.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:27 smithi042 ceph-mon[108570]: pgmap v563: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:44:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:27 smithi042 ceph-mon[113926]: pgmap v563: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:44:28.110 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.198 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:28.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:27 smithi159 ceph-mon[102519]: pgmap v563: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:44:28.299 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:28.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:28.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:28.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:28.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:28.614 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.653 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:28.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.761 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:28.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:28.801 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:28.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:28.936 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:28.937 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:44:28.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:28.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:29.039 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:29.044 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2472242715' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2472242715' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:29.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:29.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:29.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:29.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2472242715' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:44:29.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:29.396 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:29.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:29.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:29.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:29.636 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:29.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:29.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:29.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:29.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:29.921 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:29.952 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:44:29.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:29.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:30.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: pgmap v564: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4257104258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: pgmap v564: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4257104258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:29] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-09-13T14:44:30.102 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:30.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:30.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:30.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: pgmap v564: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-13T14:44:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:44:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T14:44:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4257104258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:44:30.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:30.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:30.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:30.474 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:30.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:30.611 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:30.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:30.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:30.775 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:30.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:30.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:30.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:30.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:31.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:31.094 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:44:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:30 smithi042 ceph-mon[113926]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:44:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:44:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:30 smithi042 ceph-mon[108570]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:44:31.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:31.234 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:31.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:44:31.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:30 smithi159 ceph-mon[102519]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T14:44:31.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:31.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:31.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:31.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:31.613 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:31.613 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:44:31.613 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:44:31.613 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:44:31.613 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:31.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:31.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:44:31.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:31.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:31.775 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:31.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:31.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:31.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:31.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:31.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:32.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:32.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:32.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:32.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:31 smithi159 ceph-mon[102519]: pgmap v567: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-09-13T14:44:32.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:31 smithi042 ceph-mon[113926]: pgmap v567: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-09-13T14:44:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:31 smithi042 ceph-mon[108570]: pgmap v567: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s 2024-09-13T14:44:32.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:32.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:44:32.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.517 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:32.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:32.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.645 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:44:32.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:32.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:32.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:32.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:32.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:32.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:33.017 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:33.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:33.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:33.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:33.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:33.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:33.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:33.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:33.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:33.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:33.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:33.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:33.683 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:44:33.683 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:33.725 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:44:33.725 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:33.759 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:44:33.759 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:33.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:33.793 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:44:33.793 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:33.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:33.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:33.831 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:44:33.831 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:33.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:44:33.861 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:33.896 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:44:33.896 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:33.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:33.929 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:44:33.929 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:33.951 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[108570]: pgmap v568: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:44:33.951 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.951 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.951 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3806319910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:33.951 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[113926]: pgmap v568: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:44:33.952 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.952 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.952 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3806319910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:33.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:33.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:33.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:33 smithi159 ceph-mon[102519]: pgmap v568: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 937 op/s 2024-09-13T14:44:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:33 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3806319910' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:34.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:34.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:34.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:34.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:34.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:34.304 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:34.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:34.377 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.403 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:34.504 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:34.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:34.643 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:34.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:34.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:34.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:34.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:34.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:34.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:34.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:35.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:35.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:35.137 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:35.137 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:35.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:35.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:35.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:35.421 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:35.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:35.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:35.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:35.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:35.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:35.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:35.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:35.835 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:35.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:35.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:35 smithi042 ceph-mon[113926]: pgmap v569: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-09-13T14:44:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:35 smithi042 ceph-mon[108570]: pgmap v569: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-09-13T14:44:35.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:35.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:35.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:35.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:35 smithi159 ceph-mon[102519]: pgmap v569: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-09-13T14:44:36.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.120 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:36.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:36.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:36.262 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:36.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:36.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:36.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:36.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:44:36.498 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:36.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:36.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:36.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:36.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:36.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:36.806 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:36.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:36.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:36.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:44:36.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:36.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:36.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:36.995 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:36.995 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:37.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:37.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.120 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:37.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:37.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:37.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:37.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:37.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:37.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:37.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:37.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:37.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:37.784 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:37.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:37.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:37.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:37.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:37.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:37.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:37 smithi159 ceph-mon[102519]: pgmap v570: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:44:38.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:38.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:37 smithi042 ceph-mon[108570]: pgmap v570: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:44:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:37 smithi042 ceph-mon[113926]: pgmap v570: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T14:44:38.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:38.230 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:38.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:38.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:38.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:38.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:38.506 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:38.608 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:38.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:38.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:38.845 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:38.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:38.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:38.887 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:44:38.887 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:44:38.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:39.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:39.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:39.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:39.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:39.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:39.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:39.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:39.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:39.389 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:39.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:39.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:39.542 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:39.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:39.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:44:39.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.736 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:39.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:39.749 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:44:39.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[113926]: pgmap v571: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2837014759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:39] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.51.0" 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[108570]: pgmap v571: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:44:39.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2837014759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:39.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:39 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:39.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:39.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:39.952 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:39.995 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:39 smithi159 ceph-mon[102519]: pgmap v571: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-13T14:44:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2837014759' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:39 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:44:40.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:40.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:40.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:40.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:40.337 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:40.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:40.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:40.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:40.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:40.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:40.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:40.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:40.772 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1914381672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:40.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:40.777 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.777 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:44:40.777 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:44:40.777 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1914381672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.777 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:40.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:40.778 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:40.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:40.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:40.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:40.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1914381672' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:44:40.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:41.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:41.051 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:41.154 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:41.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:41.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:41.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:41.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:41.428 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:41.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:41.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:41.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:41.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:41.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:41.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:41.745 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:41.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:41.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:41.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:41.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:41.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:41.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[113926]: pgmap v573: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-13T14:44:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:44:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[113926]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:44:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[108570]: pgmap v573: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-13T14:44:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:44:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:42.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:41 smithi042 ceph-mon[108570]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:44:42.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:42.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:42.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:42.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:42.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:41 smithi159 ceph-mon[102519]: pgmap v573: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-13T14:44:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:41 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:44:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:41 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:44:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:41 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:44:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:41 smithi159 ceph-mon[102519]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T14:44:42.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:42.379 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:42.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:42.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:42.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:42.637 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T14:44:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:42.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:42.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:42.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:42.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:42.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:42.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:42.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:43.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.062 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:43.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:43.163 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:43.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:43.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:43.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:43.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:43.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:43.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:43.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:43.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:43.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:43.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:43.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.758 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:43.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:43.828 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:43.854 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:44:43.854 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:43.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:43.893 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:44:43.893 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:43.895 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:43 smithi042 ceph-mon[108570]: pgmap v575: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:44:43.895 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:43 smithi042 ceph-mon[113926]: pgmap v575: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:44:43.910 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:43.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:43.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:43.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:44:43.929 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:43.931 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:43 smithi159 ceph-mon[102519]: pgmap v575: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-09-13T14:44:43.963 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:44:43.963 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:44.000 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:44:44.000 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:44.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:44.025 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:44:44.033 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:44:44.033 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:44.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:44.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:44.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:44:44.069 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:44.081 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:44.081 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:44:44.081 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:44:44.081 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:44:44.081 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:44.101 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:44:44.102 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:44.149 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:44.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:44.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:44.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:44.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:44.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:44.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:44.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:44.615 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:44.647 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:44.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:44.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:44.789 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:44.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:44:44.891 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:44.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:44.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:44.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:45.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3092430636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3092430636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:45.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:45.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:45.187 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:45.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:45.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3092430636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:44:45.249 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:45.350 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:45.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:45.494 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:45.531 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:45.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:45.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:45.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:45.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:45.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:45.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:45.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:45.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:45.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:45 smithi159 ceph-mon[102519]: pgmap v576: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:44:46.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:45 smithi042 ceph-mon[108570]: pgmap v576: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:44:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:45 smithi042 ceph-mon[113926]: pgmap v576: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-13T14:44:46.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:46.295 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:46.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:46.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:46.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.640 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:46.741 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:46.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:46.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:46.880 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:46.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:46.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:46.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:44:47.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.089 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:47.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:47.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:47.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:47.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:47.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.387 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:47.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:47.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:47.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:47.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:47.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:47.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:47.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:47.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:47.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:47.802 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T14:44:47.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:47.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:47.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:47.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:47.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:48.028 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:47 smithi159 ceph-mon[102519]: pgmap v577: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:44:48.029 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3932843007' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.029 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:47 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:48.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[113926]: pgmap v577: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3932843007' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[108570]: pgmap v577: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3932843007' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:47 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T14:44:48.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:89534 8.9 8:95fde08f:::benchmark_data_smithi042_130700_object89533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:36:03.147681+0000", 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 525.03792685300004, 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.150888254, 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.191 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 89534 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147681+0000", 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147681+0000", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147680+0000", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147735+0000", 2024-09-13T14:44:48.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5494000000000001e-05 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147747+0000", 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1246e-05 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147754+0000", 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.029e-06 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147805+0000", 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1233000000000002e-05 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.147935+0000", 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013041999999999999 2024-09-13T14:44:48.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.148027+0000", 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2020000000000003e-05 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.150544+0000", 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002516725 2024-09-13T14:44:48.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.150795+0000", 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00025084499999999997 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.298457+0000", 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14766249400000001 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.298527+0000", 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9079999999999996e-05 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.298570+0000", 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3087000000000003e-05 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:89539 8.9 8:9b23b7a7:::benchmark_data_smithi042_130700_object89538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:44:48.198 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:36:03.151859+0000", 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 525.03374888500002, 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16240269900000001, 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 89539 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151859+0000", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151859+0000", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151858+0000", 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151964+0000", 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000105843 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151973+0000", 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9500000000000007e-06 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.151979+0000", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0209999999999996e-06 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.152020+0000", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1718000000000003e-05 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.152140+0000", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011974800000000001 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.152227+0000", 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7433000000000001e-05 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.154553+0000", 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002325906 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.154967+0000", 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041410699999999998 2024-09-13T14:44:48.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314168+0000", 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.15920004200000001 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314220+0000", 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2800000000000003e-05 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314262+0000", 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1711000000000001e-05 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:89559 8.9 8:91819363:::benchmark_data_smithi042_130700_object89558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:36:03.176178+0000", 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 525.00943066499997, 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13822458300000001, 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.205 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 89559 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176178+0000", 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176178+0000", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176175+0000", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176246+0000", 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0883999999999994e-05 2024-09-13T14:44:48.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176266+0000", 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9715e-05 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176275+0000", 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1239999999999994e-06 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176323+0000", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8078000000000003e-05 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176470+0000", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014719099999999999 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.176620+0000", 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014936300000000001 2024-09-13T14:44:48.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.181785+0000", 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.005164887 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.181796+0000", 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1141000000000001e-05 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314315+0000", 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13251919000000001 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314364+0000", 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8609000000000002e-05 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314402+0000", 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8504999999999998e-05 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.211 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:89560 8.9 8:974c07ef:::benchmark_data_smithi042_130700_object89559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:36:03.178461+0000", 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 525.00714734200005, 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.136041833, 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.212 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 89560 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178461+0000", 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178461+0000", 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178459+0000", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178533+0000", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3113000000000003e-05 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178549+0000", 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.628e-05 2024-09-13T14:44:48.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.178556+0000", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4039999999999998e-06 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.179401+0000", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000844591 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.179539+0000", 2024-09-13T14:44:48.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000138462 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.179697+0000", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015788999999999999 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.181917+0000", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002219548 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.182534+0000", 2024-09-13T14:44:48.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00061744199999999995 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314451+0000", 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13191645499999999 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314482+0000", 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1553000000000001e-05 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:03.314503+0000", 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0511000000000001e-05 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:96306 8.9 8:93e3fd61:::benchmark_data_smithi042_130700_object96305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e111)", 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:36:10.231347+0000", 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 517.95426091700006, 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.142522704, 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.218 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 96306 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231347+0000", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231347+0000", 2024-09-13T14:44:48.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231346+0000", 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231418+0000", 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1972999999999997e-05 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231430+0000", 2024-09-13T14:44:48.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1946999999999999e-05 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.231438+0000", 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.239699+0000", 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0082609239999999994 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.239823+0000", 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000123683 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.243147+0000", 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033245129999999999 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.246666+0000", 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035185659999999999 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.247328+0000", 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00066197000000000001 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.373805+0000", 2024-09-13T14:44:48.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.12647709200000001 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.373842+0000", 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6934000000000002e-05 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:36:10.373870+0000", 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8226999999999999e-05 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.225 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:189451 8.9 8:9857ff00:::benchmark_data_smithi042_130700_object64862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:38:06.888992+0000", 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 401.29661655299998, 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13398331299999999, 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.226 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 189451 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.888992+0000", 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.888992+0000", 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.888991+0000", 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.888994+0000", 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.819e-06 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.889000+0000", 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9519999999999999e-06 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.889005+0000", 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4519999999999999e-06 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.891224+0000", 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022191509999999999 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.891352+0000", 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000127985 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.891474+0000", 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000122533 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.896051+0000", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045767250000000002 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:06.897033+0000", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00098205499999999991 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.022808+0000", 2024-09-13T14:44:48.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.12577466400000001 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.022934+0000", 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012662799999999999 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.022975+0000", 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0738000000000002e-05 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:190057 8.9 8:9fbf560b:::benchmark_data_smithi042_130700_object65468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:38:07.862831+0000", 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 400.32277673700003, 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14782505300000001, 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 190057 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862831+0000", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862831+0000", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862830+0000", 2024-09-13T14:44:48.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862836+0000", 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2099999999999998e-06 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862848+0000", 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1824999999999999e-05 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862855+0000", 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7070000000000003e-06 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.862891+0000", 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6077e-05 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.863013+0000", 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012174 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.863135+0000", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012214499999999999 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.870715+0000", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0075802500000000002 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.870754+0000", 2024-09-13T14:44:48.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9483000000000001e-05 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.010565+0000", 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.139810825 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.010615+0000", 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0229999999999998e-05 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.010657+0000", 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1071e-05 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:190058 8.9 8:9d32e57d:::benchmark_data_smithi042_130700_object65469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:38:07.865155+0000", 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 400.32045286699997, 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.17568692699999999, 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.239 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 190058 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865155+0000", 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865155+0000", 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865154+0000", 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865160+0000", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2700000000000001e-06 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865176+0000", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5716e-05 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.865184+0000", 2024-09-13T14:44:48.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8099999999999998e-06 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.866915+0000", 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0017309859999999999 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.867055+0000", 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014044700000000001 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.867194+0000", 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013875500000000001 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.870805+0000", 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036109739999999999 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:07.871691+0000", 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00088612599999999999 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.040736+0000", 2024-09-13T14:44:48.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16904454099999999 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.040799+0000", 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3948000000000005e-05 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:08.040842+0000", 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2826e-05 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:191656 8.9 8:9bc75614:::benchmark_data_smithi042_130700_object67067:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:38:10.385929+0000", 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 397.79967909099997, 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14586484499999999, 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 191656 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385929+0000", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385929+0000", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385928+0000", 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385933+0000", 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3460000000000004e-06 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385943+0000", 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0468e-05 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.385949+0000", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.161e-06 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.390268+0000", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004318835 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.390376+0000", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000108108 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.390485+0000", 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000108415 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.400361+0000", 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0098766110000000004 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.408116+0000", 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0077551399999999998 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531663+0000", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.12354620700000001 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531708+0000", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5510000000000003e-05 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531794+0000", 2024-09-13T14:44:48.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5916999999999997e-05 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34323.0:191663 8.9 8:9d164ff0:::benchmark_data_smithi042_130700_object67074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:38:10.395326+0000", 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 397.79028249999999, 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13657847200000001, 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34323", 2024-09-13T14:44:48.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 191663 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395326+0000", 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395326+0000", 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395324+0000", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395332+0000", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2179999999999999e-06 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395347+0000", 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4768e-05 2024-09-13T14:44:48.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395355+0000", 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0020000000000006e-06 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395773+0000", 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041758799999999999 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395883+0000", 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110814 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.395993+0000", 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010940799999999999 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.400447+0000", 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004454141 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.408144+0000", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0076975619999999998 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531835+0000", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.12369047599999999 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531873+0000", 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8581999999999998e-05 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:38:10.531904+0000", 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0703000000000001e-05 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:14854 9.0 e143/139 9:0d9e4836:::benchmark_data_smithi042_157727_object14853:head v 143'923, mlcod=143'919)", 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:42:21.126907+0000", 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 147.05870090600001, 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.046989123000000001, 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.259 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126907+0000", 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126907+0000", 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126906+0000", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126979+0000", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3447000000000003e-05 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126991+0000", 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1871e-05 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.126998+0000", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6329999999999999e-06 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127039+0000", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0552000000000001e-05 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127057+0000", 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8077e-05 2024-09-13T14:44:48.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.128261+0000", 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0012047119999999999 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173896+0000", 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.045635109 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:14865 9.1 e143/139 9:83ef43b3:::benchmark_data_smithi042_157727_object14864:head v 143'923, mlcod=143'921)", 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:42:21.127732+0000", 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 147.057876284, 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.046199853999999999, 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127732+0000", 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127732+0000", 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127731+0000", 2024-09-13T14:44:48.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:48.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:48.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127772+0000", 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1085999999999999e-05 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127784+0000", 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1678999999999999e-05 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127790+0000", 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8869999999999997e-06 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127832+0000", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2290000000000003e-05 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.127851+0000", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8597000000000002e-05 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173898+0000", 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.046047178000000001 2024-09-13T14:44:48.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173932+0000", 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3908999999999997e-05 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:14872 9.0 e143/139 9:034e37f6:::benchmark_data_smithi042_157727_object14871:head v 143'924, mlcod=143'921)", 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:42:21.131281+0000", 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 147.05432677100001, 2024-09-13T14:44:48.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.044960804, 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131281+0000", 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131281+0000", 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131280+0000", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131318+0000", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7963000000000002e-05 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173873+0000", 2024-09-13T14:44:48.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.042554900999999999 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173887+0000", 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3665999999999999e-05 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173923+0000", 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5904e-05 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173941+0000", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8184000000000001e-05 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.176208+0000", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002267373 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.176242+0000", 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4099e-05 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.274 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:14880 9.b e143/139 9:da9b7476:::benchmark_data_smithi042_157727_object14879:head v 143'946, mlcod=143'944)", 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:42:21.131397+0000", 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 147.054210794, 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.044464482, 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131397+0000", 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131397+0000", 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131397+0000", 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.131473+0000", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6794999999999999e-05 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173880+0000", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.042406465999999997 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.173889+0000", 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6539999999999994e-06 2024-09-13T14:44:48.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.174013+0000", 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000123991 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.174072+0000", 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8717e-05 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.175825+0000", 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0017524000000000001 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:21.175862+0000", 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7338000000000001e-05 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:168339 9.b e162/139 9:d7813a5e:::benchmark_data_smithi042_157727_object38390:head v 162'10587, mlcod=162'10584)", 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:26.129637+0000", 2024-09-13T14:44:48.279 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.055970864999999, 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.031142691, 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129637+0000", 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129637+0000", 2024-09-13T14:44:48.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129636+0000", 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129646+0000", 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4129999999999995e-06 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129663+0000", 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7048e-05 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129672+0000", 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9989999999999997e-06 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129881+0000", 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020917599999999999 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.129934+0000", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3065000000000001e-05 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.160747+0000", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.030813562999999999 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:26.160780+0000", 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2579000000000001e-05 2024-09-13T14:44:48.283 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:172473 9.b e164/139 9:d15c8ea1:::benchmark_data_smithi042_157727_object42524:head v 164'10851, mlcod=164'10847)", 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:30.652553+0000", 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 17.533055413, 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026904015, 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.284 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652553+0000", 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652553+0000", 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652552+0000", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652559+0000", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5000000000000002e-06 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652574+0000", 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5056e-05 2024-09-13T14:44:48.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652581+0000", 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9700000000000002e-06 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652625+0000", 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3207000000000002e-05 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.652645+0000", 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0735999999999999e-05 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.679344+0000", 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026698715000000001 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:30.679457+0000", 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112871 2024-09-13T14:44:48.288 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:175407 9.b e164/139 9:dfa6edbe:::benchmark_data_smithi042_157727_object45458:head v 164'11068, mlcod=164'11066)", 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:33.831297+0000", 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 14.354311253000001, 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027132949999999999, 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831297+0000", 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831297+0000", 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831292+0000", 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999952 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831305+0000", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2378000000000001e-05 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831325+0000", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0364e-05 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831333+0000", 2024-09-13T14:44:48.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5809999999999999e-06 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831398+0000", 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5123999999999995e-05 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.831423+0000", 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4905e-05 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.858250+0000", 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026827186999999999 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:33.858430+0000", 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018009999999999999 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:187158 9.b e166/139 9:d9c478d4:::benchmark_data_smithi042_157727_object57209:head v 166'11839, mlcod=166'11837)", 2024-09-13T14:44:48.293 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:46.067764+0000", 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 2.117844335, 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.024539453999999999, 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067764+0000", 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067764+0000", 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067763+0000", 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067772+0000", 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5319999999999995e-06 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067793+0000", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1277e-05 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067803+0000", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.3729999999999999e-06 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067864+0000", 2024-09-13T14:44:48.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1295000000000002e-05 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.067887+0000", 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3359000000000001e-05 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.092192+0000", 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.024304413 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.092303+0000", 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111544 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:187307 9.b e166/139 9:d903fbb0:::benchmark_data_smithi042_157727_object57358:head v 166'11853, mlcod=166'11851)", 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:46.236675+0000", 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 1.948933061, 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.024252847000000001, 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.298 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236675+0000", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236675+0000", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236675+0000", 2024-09-13T14:44:48.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236680+0000", 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4929999999999998e-06 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236688+0000", 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.357e-06 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.236692+0000", 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3970000000000004e-06 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.240544+0000", 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038516140000000002 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.240588+0000", 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4161e-05 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.260891+0000", 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.020303774 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:46.260928+0000", 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6560999999999998e-05 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:187954 9.b e166/139 9:d53a739f:::benchmark_data_smithi042_157727_object58005:head v 166'11897, mlcod=166'11892)", 2024-09-13T14:44:48.302 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:47.064432+0000", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 1.121176392, 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022477896000000001, 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064432+0000", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064432+0000", 2024-09-13T14:44:48.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064432+0000", 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064438+0000", 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3709999999999996e-06 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064443+0000", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7600000000000002e-06 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.064446+0000", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4879999999999999e-06 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.073696+0000", 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0092500480000000003 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.073790+0000", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.3877999999999998e-05 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.086893+0000", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013102793999999999 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:47.086910+0000", 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6748e-05 2024-09-13T14:44:48.306 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.307 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.307 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.307 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:48.307 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:48.307 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:48.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:48.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:48.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:48.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:48.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:48.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:48.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:48.670 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:48.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:48.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:48.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:48.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:48.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:48.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:48.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:48.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:48 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:44:48.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:48 smithi159 ceph-mon[102519]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:44:49.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:49.086 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:49.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:49.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:48 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:44:49.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:48 smithi042 ceph-mon[113926]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:44:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:48 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T14:44:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:48 smithi042 ceph-mon[108570]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T14:44:49.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:49.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:44:49.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.225 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:49.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:49.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:49.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:49.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:49.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:49.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:49.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:49.566 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:49.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:49.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:49.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.807 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:49.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:49.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:49.960 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:49.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:49.961 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:49 smithi042 ceph-mon[113926]: pgmap v579: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:49.961 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:49 smithi042 ceph-mon[108570]: pgmap v579: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:49.962 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:49] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.51.0" 2024-09-13T14:44:49.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:49 smithi159 ceph-mon[102519]: pgmap v579: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:50.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:50.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:50.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:50.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:50.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:50.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:50.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:44:50.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.369 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:50.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:50.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.524 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:50.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:50.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:50.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:50.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:50.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:50.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:50.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:50.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:50.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:50.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:51.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:51.107 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:51.209 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:51.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:51.347 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.522 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:51.529 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:51.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.583 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:51.583 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:44:51.583 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:192112 9.a e167/139 9:5004108b:::benchmark_data_smithi042_157727_object62163:head v 167'12055, mlcod=167'12053)", 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:51.579672+0000", 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0023993349999999998, 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024327070000000001, 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579672+0000", 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:51.584 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579672+0000", 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579671+0000", 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:51.585 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579679+0000", 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5560000000000008e-06 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.586 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.587 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:51.587 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579690+0000", 2024-09-13T14:44:51.587 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0934000000000001e-05 2024-09-13T14:44:51.587 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.587 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.588 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:51.588 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579698+0000", 2024-09-13T14:44:51.588 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7239999999999992e-06 2024-09-13T14:44:51.588 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.588 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579740+0000", 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1816000000000001e-05 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.589 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.590 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:51.590 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.579761+0000", 2024-09-13T14:44:51.590 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0894999999999999e-05 2024-09-13T14:44:51.590 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:51.591 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:51.591 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:51.591 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.591 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:192107 9.9 e167/139 9:92805cc5:::benchmark_data_smithi042_157727_object62158:head v 167'11977, mlcod=167'11974)", 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:51.581502+0000", 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00056921000000000001, 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00065204599999999998, 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581502+0000", 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.592 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581502+0000", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581501+0000", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581507+0000", 2024-09-13T14:44:51.593 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0299999999999999e-06 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581529+0000", 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1974999999999999e-05 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581535+0000", 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2339999999999999e-06 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:51.594 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581616+0000", 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0311999999999999e-05 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:51.581635+0000", 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.967e-05 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 2 2024-09-13T14:44:51.595 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:51.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:51.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:51.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:51.825 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:44:51.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:51.956 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:51.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:51.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:51 smithi159 ceph-mon[102519]: pgmap v580: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:44:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:51.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1790649301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:51 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:52.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:52.097 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[113926]: pgmap v580: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:44:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1790649301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[108570]: pgmap v580: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 845 op/s 2024-09-13T14:44:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:44:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:44:52.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1790649301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:52.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:51 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:44:52.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:52.236 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:52.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:52.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:52.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:52.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:52.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:52.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:52.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:44:52.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:52.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:52.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:52.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:52.763 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:52.798 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:52.841 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:52.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:52.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:52.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:53.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:53.046 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:52 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:53.046 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:52 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:53.046 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:52 smithi159 ceph-mon[102519]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:44:53.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.087 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[108570]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:44:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:52 smithi042 ceph-mon[113926]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T14:44:53.188 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:53.331 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:53.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:53.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:53.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:53.643 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:53.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:53.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:53.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:53.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:53.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:54.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:54.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:54.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:53 smithi042 ceph-mon[113926]: pgmap v582: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:44:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:53 smithi042 ceph-mon[108570]: pgmap v582: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:44:54.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.246 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:53 smithi159 ceph-mon[102519]: pgmap v582: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T14:44:54.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:54.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:54.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:54.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:54.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:54.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:54.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:54.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:54.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:54.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:54.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:54.962 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:54.962 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:44:54.962 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:44:54.962 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:44:54.962 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:55.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.050 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:55.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:55.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:55.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:55.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:55.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:55.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:55.430 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.479 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:55.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:55.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:55.628 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:55.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:44:55.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:55.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:55.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:55.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:55.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:55.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:55.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:55 smithi159 ceph-mon[102519]: pgmap v583: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:44:56.006 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:55 smithi042 ceph-mon[113926]: pgmap v583: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:44:56.006 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:55 smithi042 ceph-mon[108570]: pgmap v583: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-09-13T14:44:56.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.029 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:44:56.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.064 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:56.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:56.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:56.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:56.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:56.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:44:56.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:44:56.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:56.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:56.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:56.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:44:56.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:44:56.713 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:56.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:56.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:56.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:44:56.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:56.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:56.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:56.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:57.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:57.075 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:57.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:57.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:57.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:57.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:57.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:57.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.500 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:57.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:57.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:57.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:57.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:57.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:57.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:57.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:57.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:57.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:57.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:58.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:57 smithi042 ceph-mon[113926]: pgmap v584: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:57 smithi042 ceph-mon[108570]: pgmap v584: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:58.113 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:57 smithi159 ceph-mon[102519]: pgmap v584: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:42453 9.3 9:c7e8501d:::benchmark_data_smithi042_157727_object42452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2024-09-13T14:44:58.157 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:42:32.989293+0000", 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 145.16281654100001, 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074031507999999996, 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 42453 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.158 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989293+0000", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989293+0000", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989292+0000", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989323+0000", 2024-09-13T14:44:58.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1075000000000003e-05 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989333+0000", 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1759999999999999e-06 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.989339+0000", 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8789999999999997e-06 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.160 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.990709+0000", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013692509999999999 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.990843+0000", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000134485 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.990975+0000", 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000131385 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.161 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.994169+0000", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0031948010000000002 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:32.996106+0000", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0019367989999999999 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:33.063227+0000", 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.067120421999999999 2024-09-13T14:44:58.162 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:33.063269+0000", 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2778000000000003e-05 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:42:33.063325+0000", 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5461999999999999e-05 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.163 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84558 9.3 9:c85b4db8:::benchmark_data_smithi042_157727_object84557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.51067098199999, 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.37831424299999999, 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84558 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.164 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641438+0000", 2024-09-13T14:44:58.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641520+0000", 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2422000000000005e-05 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.166 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641531+0000", 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0536e-05 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641537+0000", 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642039+0000", 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050194900000000001 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642168+0000", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012866399999999999 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642290+0000", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012242000000000001 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.647958+0000", 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056680840000000003 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719589+0000", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.071630894000000001 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019655+0000", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30006546099999998 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019711+0000", 2024-09-13T14:44:58.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6632000000000003e-05 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019753+0000", 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1925999999999998e-05 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84559 9.3 9:c8ad0e78:::benchmark_data_smithi042_157727_object84558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:44:58.170 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.510523592, 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.38015051100000002, 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84559 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:44:58.171 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.172 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641983+0000", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039788200000000001 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642357+0000", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00037351099999999998 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642364+0000", 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2790000000000001e-06 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645661+0000", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032972330000000001 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645784+0000", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012222900000000001 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645882+0000", 2024-09-13T14:44:58.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8276999999999994e-05 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.648108+0000", 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022264009999999998 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.722650+0000", 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074541741999999994 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021646+0000", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29899620700000001 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021712+0000", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5488999999999996e-05 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021737+0000", 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4961e-05 2024-09-13T14:44:58.176 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84563 9.d 9:ba52b3ee:::benchmark_data_smithi042_157727_object84562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.43234515100001, 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.31030390200000002, 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.177 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84563 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719763+0000", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719824+0000", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0915999999999997e-05 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719836+0000", 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1992e-05 2024-09-13T14:44:58.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719841+0000", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.271e-06 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719885+0000", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3130000000000002e-05 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719994+0000", 2024-09-13T14:44:58.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109428 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.720091+0000", 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7001000000000002e-05 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723173+0000", 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030821899999999998 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723401+0000", 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022781400000000001 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.029965+0000", 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30656361599999998 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030017+0000", 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2584999999999999e-05 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030069+0000", 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1329999999999998e-05 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84572 9.3 9:cea63616:::benchmark_data_smithi042_157727_object84571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.424508622, 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30071340699999999, 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.183 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84572 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:44:58.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.185 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:58.186 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727600+0000", 2024-09-13T14:44:58.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.186 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.186 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727712+0000", 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112267 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727728+0000", 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6545000000000002e-05 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727736+0000", 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7950000000000008e-06 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727777+0000", 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0595e-05 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727921+0000", 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000143879 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728044+0000", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000123196 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730658+0000", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026146229999999999 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731148+0000", 2024-09-13T14:44:58.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048908600000000001 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028092+0000", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29694452599999999 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028269+0000", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000177126 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028315+0000", 2024-09-13T14:44:58.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5414999999999998e-05 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84573 9.e 9:7570e4c0:::benchmark_data_smithi042_157727_object84572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.42432002699999, 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29393596500000002, 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.191 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84573 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727789+0000", 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727845+0000", 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5803000000000001e-05 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727853+0000", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3659999999999992e-06 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727857+0000", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9099999999999998e-06 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727888+0000", 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0574999999999998e-05 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727979+0000", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1242999999999994e-05 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728059+0000", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0238e-05 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730776+0000", 2024-09-13T14:44:58.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002716684 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730964+0000", 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018814 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021615+0000", 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29065137200000002 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021666+0000", 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0467000000000003e-05 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021726+0000", 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9982000000000003e-05 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84574 9.d 9:bb2c6928:::benchmark_data_smithi042_157727_object84573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:44:58.197 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 116.421007993, 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29919750000000001, 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84574 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:44:58.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731101+0000", 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731161+0000", 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0677999999999997e-05 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731171+0000", 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7480000000000005e-06 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731177+0000", 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731204+0000", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7103000000000001e-05 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731302+0000", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7919999999999998e-05 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731399+0000", 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6759e-05 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733667+0000", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022682940000000001 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733966+0000", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000298528 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030141+0000", 2024-09-13T14:44:58.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29617486399999998 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030264+0000", 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012284000000000001 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030299+0000", 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5714999999999999e-05 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111462 9.d 9:be6e2a9e:::benchmark_data_smithi042_157727_object111461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 91.576809772999994, 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.090455042999999999, 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111462 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575299+0000", 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575330+0000", 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1720000000000001e-05 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575344+0000", 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3326000000000001e-05 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575350+0000", 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7780000000000002e-06 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575386+0000", 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5271000000000002e-05 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575538+0000", 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015224600000000001 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575677+0000", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000138925 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.579009+0000", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00333228 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.579562+0000", 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000552773 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665646+0000", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.086084385999999999 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665695+0000", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8665999999999998e-05 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665755+0000", 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0127000000000002e-05 2024-09-13T14:44:58.209 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111463 9.3 9:c145cb7d:::benchmark_data_smithi042_157727_object111462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 91.576700264999999, 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.086132666999999996, 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.210 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111463 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575409+0000", 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575476+0000", 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7002999999999995e-05 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575484+0000", 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3489999999999992e-06 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575490+0000", 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7660000000000004e-06 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575529+0000", 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9245000000000001e-05 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575610+0000", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0665000000000001e-05 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575695+0000", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4841999999999998e-05 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.578801+0000", 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003106765 2024-09-13T14:44:58.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.582252+0000", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034510109999999999 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661446+0000", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.079193685999999999 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661489+0000", 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2667000000000003e-05 2024-09-13T14:44:58.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661542+0000", 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3461999999999997e-05 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111470 9.d 9:b5f0614c:::benchmark_data_smithi042_157727_object111469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.581297+0000", 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 91.570812750000002, 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.084566805999999994, 2024-09-13T14:44:58.216 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111470 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581297+0000", 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581297+0000", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581296+0000", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581347+0000", 2024-09-13T14:44:58.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1031e-05 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581357+0000", 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0801e-05 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581363+0000", 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9179999999999999e-06 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581402+0000", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8948e-05 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581551+0000", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014826600000000001 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.581725+0000", 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017463 2024-09-13T14:44:58.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.586945+0000", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0052196860000000003 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588063+0000", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001117741 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665794+0000", 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.077731049999999996 2024-09-13T14:44:58.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665834+0000", 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0407999999999997e-05 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665864+0000", 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9867999999999999e-05 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111477 9.e 9:7ca6fc8e:::benchmark_data_smithi042_157727_object111476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 91.563887569000002, 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.085265375000000004, 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111477 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588221+0000", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588281+0000", 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0575000000000002e-05 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588294+0000", 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2473000000000001e-05 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588298+0000", 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7929999999999997e-06 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588330+0000", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1347999999999999e-05 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588456+0000", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012629699999999999 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588548+0000", 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1544000000000003e-05 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.591582+0000", 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030342239999999999 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.592407+0000", 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00082471300000000003 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673441+0000", 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.081034410000000001 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673469+0000", 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7577e-05 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673488+0000", 2024-09-13T14:44:58.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9074999999999999e-05 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111478 9.3 9:c13653fd:::benchmark_data_smithi042_157727_object111477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 91.563765008999994, 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.096331477999999998, 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.230 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111478 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588344+0000", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588419+0000", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4846999999999997e-05 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588428+0000", 2024-09-13T14:44:58.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7290000000000006e-06 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588434+0000", 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2820000000000002e-06 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588480+0000", 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5836e-05 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588639+0000", 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015876700000000001 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588790+0000", 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015128799999999999 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.591754+0000", 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0029641699999999999 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.672268+0000", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.080513578000000002 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684565+0000", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012296860999999999 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684637+0000", 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2665999999999998e-05 2024-09-13T14:44:58.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684676+0000", 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9135999999999999e-05 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111884 9.3 9:c842fdc2:::benchmark_data_smithi042_157727_object111883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 90.802570040000006, 2024-09-13T14:44:58.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.108810141, 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111884 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349538+0000", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349608+0000", 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0049000000000006e-05 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349624+0000", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5537999999999999e-05 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349631+0000", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349675+0000", 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3405000000000002e-05 2024-09-13T14:44:58.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349870+0000", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000195513 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.350096+0000", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000225796 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.354559+0000", 2024-09-13T14:44:58.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044625400000000001 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.354567+0000", 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4570000000000005e-06 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458271+0000", 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.103703352 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458315+0000", 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4193000000000002e-05 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458350+0000", 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5204999999999999e-05 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111892 9.3 9:cc204ca5:::benchmark_data_smithi042_157727_object111891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:44:58.242 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 90.794788234999999, 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.106581274, 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111892 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:44:58.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357320+0000", 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357396+0000", 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5364000000000006e-05 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357408+0000", 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2428e-05 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357413+0000", 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3120000000000001e-06 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357455+0000", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.193e-05 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357560+0000", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000105042 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357679+0000", 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011904399999999999 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.360725+0000", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030456329999999998 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.361549+0000", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00082388600000000002 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463812+0000", 2024-09-13T14:44:58.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.102262805 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463861+0000", 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9280000000000003e-05 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463903+0000", 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1993000000000003e-05 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:112241 9.e 9:791f35e8:::benchmark_data_smithi042_157727_object112240:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.753685+0000", 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 90.39842496, 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.073349866, 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 112241 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753685+0000", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753685+0000", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753684+0000", 2024-09-13T14:44:58.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753785+0000", 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000101284 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753799+0000", 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4344e-05 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753806+0000", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6930000000000001e-06 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753857+0000", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1505000000000002e-05 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.753995+0000", 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013750700000000001 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.754132+0000", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000137427 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.759863+0000", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057310319999999996 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.759889+0000", 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5292000000000001e-05 2024-09-13T14:44:58.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.826944+0000", 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.067055058000000001 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.826999+0000", 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4905999999999999e-05 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.827035+0000", 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6191e-05 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:112242 9.e 9:7ab2adf2:::benchmark_data_smithi042_157727_object112241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.756083+0000", 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 90.396026479, 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.078771718000000004, 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:44:58.255 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 112242 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756083+0000", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756083+0000", 2024-09-13T14:44:58.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756082+0000", 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756130+0000", 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8251000000000003e-05 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756143+0000", 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2918000000000001e-05 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.756150+0000", 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2420000000000003e-06 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.757392+0000", 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00124142 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.757503+0000", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011150399999999999 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.757636+0000", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013308600000000001 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.759997+0000", 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023608050000000001 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.760404+0000", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00040714199999999999 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.834792+0000", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074387629999999996 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.834828+0000", 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5920000000000002e-05 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.834855+0000", 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7288e-05 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:169880 9.9 e162/139 9:9fd111e2:::benchmark_data_smithi042_157727_object39931:head v 162'10566, mlcod=162'10564)", 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:27.745457+0000", 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 30.406653228, 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.075570222000000006, 2024-09-13T14:44:58.261 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745457+0000", 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745457+0000", 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745456+0000", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745460+0000", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8190000000000002e-06 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745465+0000", 2024-09-13T14:44:58.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.395e-06 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745469+0000", 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.326e-06 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745500+0000", 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1492999999999998e-05 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745518+0000", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7513999999999999e-05 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.748956+0000", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034381199999999998 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.821027+0000", 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.072071031999999993 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.265 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:169883 9.5 e162/139 9:a563a887:::benchmark_data_smithi042_157727_object39934:head v 162'10624, mlcod=162'10622)", 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:27.745692+0000", 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 30.406417352999998, 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074505699999999994, 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745692+0000", 2024-09-13T14:44:58.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745692+0000", 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745692+0000", 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745697+0000", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7140000000000001e-06 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745705+0000", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0770000000000008e-06 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745710+0000", 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8600000000000001e-06 2024-09-13T14:44:58.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745741+0000", 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1467000000000002e-05 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.745754+0000", 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.324e-05 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820159+0000", 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074404734 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820198+0000", 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9168999999999997e-05 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:169886 9.1 e162/139 9:89ca26e9:::benchmark_data_smithi042_157727_object39937:head v 162'10633, mlcod=162'10631)", 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:27.749182+0000", 2024-09-13T14:44:58.270 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 30.402927943000002, 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.085086185999999994, 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.749182+0000", 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.749182+0000", 2024-09-13T14:44:58.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.749181+0000", 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.749188+0000", 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5690000000000001e-06 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.825838+0000", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.076650100999999998 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.825848+0000", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0733999999999999e-05 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.825901+0000", 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2732999999999998e-05 2024-09-13T14:44:58.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.825925+0000", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3302000000000002e-05 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.834231+0000", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0083067990000000001 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.834268+0000", 2024-09-13T14:44:58.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6767000000000002e-05 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:169887 9.e e162/156 9:72bcb8fb:::benchmark_data_smithi042_157727_object39938:head v 162'10649, mlcod=162'10647)", 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:44:27.752072+0000", 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 30.400038017, 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.072872082000000005, 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.752072+0000", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.752072+0000", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.752071+0000", 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:44:58.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.752085+0000", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3573e-05 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820153+0000", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.068068742000000002 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820164+0000", 2024-09-13T14:44:58.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0407999999999999e-05 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820204+0000", 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9767000000000001e-05 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.820226+0000", 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2654e-05 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.824845+0000", 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046188169999999999 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:44:27.824944+0000", 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8849000000000002e-05 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:44:58.279 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:44:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.281 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:58.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:58.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:58.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:58.525 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:44:58.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:58.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:44:58.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:44:58.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:58.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:44:58.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:58.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:58.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:58.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:44:58.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:58.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:44:58.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:44:59.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:44:59.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:44:59.113 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:44:59.113 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:44:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:44:59.151 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:44:59.152 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:44:59.194 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:44:59.194 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:44:59.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:59.236 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:44:59.236 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:44:59.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:44:59.273 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:44:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.273 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:59.311 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:44:59.312 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:44:59.350 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:44:59.351 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:44:59.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:44:59.387 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:44:59.387 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:44:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:44:59.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:59.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:59.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:59.722 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:44:59.757 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:59.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:44:59.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:44:59.898 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:44:59.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:44:59.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:00.031 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:59 smithi042 ceph-mon[108570]: pgmap v585: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:44:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3317616273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:59 smithi042 ceph-mon[113926]: pgmap v585: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:44:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3317616273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:44:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:44:59] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-09-13T14:45:00.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:00.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:59 smithi159 ceph-mon[102519]: pgmap v585: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:44:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3317616273' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:00.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:00.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:00.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:00.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.464 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:00.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:00.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:00.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:00.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:00.697 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:00.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:00.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:00.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:00.864 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:45:00.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:00.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:00.992 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:01.029 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:01.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2674616237' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2674616237' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.130 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:01.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:01.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2674616237' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:45:01.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:01.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:01.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:01.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:01.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:01.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:01.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:01.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:01.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:01.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.729 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:01.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:01.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:01.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:01.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:01.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:02.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.021 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[113926]: pgmap v586: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:45:02.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[113926]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:45:02.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[108570]: pgmap v586: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:02.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:45:02.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:01 smithi042 ceph-mon[108570]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:45:02.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:02.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:01 smithi159 ceph-mon[102519]: pgmap v586: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:01 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:45:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:01 smithi159 ceph-mon[102519]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T14:45:02.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:02.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:02.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:02.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:45:02.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:02.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:02.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:02.565 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:02.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:02.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:02.893 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:02.922 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:02.965 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:02.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:03.038 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:03.038 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:03.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:03.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:03.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:03.219 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:03.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:03.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:03.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:03.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:03.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.538 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:03.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:03.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:03.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.673 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:03.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:03.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:03.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:03.866 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:45:03.912 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:03.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:03.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:04.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:04.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:04.186 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:04.223 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:03 smithi159 ceph-mon[102519]: pgmap v588: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:45:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/915129123' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[113926]: pgmap v588: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/915129123' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[108570]: pgmap v588: 99 pgs: 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/915129123' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:04.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:04.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:04.638 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:04.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:04.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:04.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:04.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:04.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:04.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:04.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:05.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:05.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:05.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:05.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:05.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:05.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[113926]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:45:05.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:05.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:05.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:04 smithi042 ceph-mon[108570]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:45:05.161 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:05.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:05.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:04 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:05.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:04 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:04 smithi159 ceph-mon[102519]: osdmap e170: 8 total, 8 up, 8 in 2024-09-13T14:45:05.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:05.344 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:05.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:05.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.493 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:05.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:05.594 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:05.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:05.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:05.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:05.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:05.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:05.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:05.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:05.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:05.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:05.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[113926]: pgmap v590: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-13T14:45:05.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:05.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:05.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[108570]: pgmap v590: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-13T14:45:05.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:05.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:06.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:06.053 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:06.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:06.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:06.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:06.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:05 smithi159 ceph-mon[102519]: pgmap v590: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-09-13T14:45:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:06.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:06.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:06.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:06.428 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:06.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:06.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:06.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:06.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:06.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:06.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:06.839 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:06.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:06.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:06.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:06.969 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:07.010 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:07.112 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:07.143 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:07.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:07.279 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:07.381 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:07.416 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:07.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:07.552 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:07.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:07.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:07.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:07.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:07.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:07.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:07.977 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:07.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:08.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:07 smithi042 ceph-mon[113926]: pgmap v591: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:45:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:07 smithi042 ceph-mon[108570]: pgmap v591: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:45:08.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:08.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:08.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:07 smithi159 ceph-mon[102519]: pgmap v591: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T14:45:08.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:08.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:08.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:08.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:08.495 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:08.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:08.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:08.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:08.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:08.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:08.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:08.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:08.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:08.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:08.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:08.994 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:08.994 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:09.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:09.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:09.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:09.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:09.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:09.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:09.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:09.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:09.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:09.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:09.589 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:45:09.598 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:09.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:09.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:09.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:09.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:09.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:09] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.51.0" 2024-09-13T14:45:09.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:09.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:09.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:10.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:10.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:10.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:10.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:10.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:09 smithi159 ceph-mon[102519]: pgmap v592: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:45:10.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2483587116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:10.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:10.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:09 smithi042 ceph-mon[108570]: pgmap v592: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:45:10.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2483587116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:10.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:09 smithi042 ceph-mon[113926]: pgmap v592: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T14:45:10.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2483587116' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.377 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:10.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:10.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:10.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:10.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:10.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:10.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:10.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:10.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:10.790 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:10.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:10.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:10.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:10.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:11.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:11.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:11.156 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.191 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:11.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:11.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:11.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:11.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:11.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:11.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:11.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:11.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:11.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:11.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:11.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:11.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:11.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:12.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.234 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:11 smithi159 ceph-mon[102519]: pgmap v593: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:12.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:12.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:11 smithi042 ceph-mon[113926]: pgmap v593: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:11 smithi042 ceph-mon[108570]: pgmap v593: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:12.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.400 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:12.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.484 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:45:12.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.578 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:12.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:12.679 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:12.814 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:45:12.814 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:45:12.814 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:45:12.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:12.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:12.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:12.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:12.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:12.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:13.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:13.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:13.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:13.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:13.374 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:13.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:13.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:13.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:13.560 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:13.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:13.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:13.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:13.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:13.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:13.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:13.945 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:13 smithi042 ceph-mon[113926]: pgmap v594: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:45:13.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:13 smithi042 ceph-mon[108570]: pgmap v594: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:45:13.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:14.012 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.033 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:14.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:14.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:14.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:14.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:13 smithi159 ceph-mon[102519]: pgmap v594: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-13T14:45:14.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:14.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:14.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:14.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:14.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:14.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:14.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:14.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:14.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:14.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:14.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:14.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:14.912 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:45:14.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:15.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:15.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4020345441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4020345441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:15.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:15.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4020345441' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:45:15.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:15.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:15.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:15.407 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:15.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.429 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:15.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:15.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:15.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.578 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:15.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:15.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.729 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:15.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:15.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:15.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:15.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:15.969 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:16.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:16.114 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:16.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:16.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:15 smithi159 ceph-mon[102519]: pgmap v595: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:45:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:45:16.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:15 smithi159 ceph-mon[102519]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:45:16.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:16.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[113926]: pgmap v595: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:45:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:45:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[113926]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:45:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[108570]: pgmap v595: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-13T14:45:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:45:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:15 smithi042 ceph-mon[108570]: osdmap e171: 8 total, 8 up, 8 in 2024-09-13T14:45:16.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:16.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:16.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:16.574 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:16.631 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:16.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:16.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:16.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:16.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:16.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:16.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:16.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:16.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:17.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:17.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:17.236 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:17.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:17.365 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:17.396 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:17.498 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:17.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:17.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:17.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:17.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:17.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:17.807 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:17.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:17.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:17.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:17.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:18.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:18.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[113926]: pgmap v597: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2222151230' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[108570]: pgmap v597: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2222151230' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:17 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:18.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:18.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:17 smithi159 ceph-mon[102519]: pgmap v597: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-09-13T14:45:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:17 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2222151230' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:17 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:45:18.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:18.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.351 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:18.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:18.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:18.488 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:18.500 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:45:18.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:18.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:18.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:18.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:18.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:18.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:18.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:18.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:19.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:19.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:19.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:19.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.197 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:18 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:18 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:18 smithi159 ceph-mon[102519]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:45:19.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:19.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[113926]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:45:19.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:18 smithi042 ceph-mon[108570]: osdmap e172: 8 total, 8 up, 8 in 2024-09-13T14:45:19.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:19.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:19.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:19.622 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:19.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:19.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.756 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:19.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:19.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:19] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.51.0" 2024-09-13T14:45:19.857 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:19.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:19.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:19.992 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:20.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:20.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:45:20.131 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:20.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:20.216 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:19 smithi042 ceph-mon[113926]: pgmap v599: 99 pgs: 99 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:45:20.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:19 smithi042 ceph-mon[108570]: pgmap v599: 99 pgs: 99 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:45:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:19 smithi159 ceph-mon[102519]: pgmap v599: 99 pgs: 99 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-13T14:45:20.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:20.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:20.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:20.464 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:20.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:20.566 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:20.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:20.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:20.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:20.737 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:20.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:20.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:20.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:20.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:20.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:20.933 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:45:20.933 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:20.973 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:45:20.973 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:20.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:21.011 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.011 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3722797755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:21.012 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.012 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3722797755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:21.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.016 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:45:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.016 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:21.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:21.051 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:45:21.052 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:21.052 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:21.052 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:21.086 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:45:21.086 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:21.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:21.121 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:45:21.121 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.157 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:21.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:21.158 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:45:21.158 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:21.190 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:45:21.190 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:21.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3722797755' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:21.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:21.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.287 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:21.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:21.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:21.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:21.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:21.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:21.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:45:21.660 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:21.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:21.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:21.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:21.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:21.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:21.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:21.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:22.069 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:22.114 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:22.216 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:22.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:21 smithi159 ceph-mon[102519]: pgmap v600: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:45:22.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:22.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:21 smithi042 ceph-mon[113926]: pgmap v600: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:45:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:21 smithi042 ceph-mon[108570]: pgmap v600: 99 pgs: 99 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-13T14:45:22.358 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.402 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:22.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:22.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:22.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:45:22.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:22.595 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:22.623 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:22.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:22.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:22.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:22.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:22.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:22.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:23.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:23.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:23.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:23.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:23.262 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:23.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:23.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:23.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:23.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:23.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:23.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:23.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:23.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:23.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:23.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:23.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:23.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:23.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:23.984 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:23.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:24.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:24.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:24.114 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:23 smithi042 ceph-mon[108570]: pgmap v601: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:24.115 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:23 smithi042 ceph-mon[113926]: pgmap v601: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:24.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:24.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:23 smithi159 ceph-mon[102519]: pgmap v601: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:24.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:24.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.310 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:24.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:24.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:24.452 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:24.554 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:24.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:24.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:24.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.732 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:24.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:24.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:24.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:24.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:24.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:24.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:25.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:25.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:25.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:25.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:25.533 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:25.578 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:25.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:25.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:25.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:25.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:25.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:25.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:25.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:25.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:25.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:26.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:26.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:26.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:26.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:25 smithi159 ceph-mon[102519]: pgmap v602: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:45:26.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:25 smithi042 ceph-mon[113926]: pgmap v602: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:45:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:25 smithi042 ceph-mon[108570]: pgmap v602: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T14:45:26.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:26.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:26.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:26.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:26.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:26.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:26.661 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:26.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:26.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:26.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:26.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:26.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:26.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:27.040 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:27.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:27.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:27.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:27.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:27.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:27.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:27.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:27.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:27.595 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:27.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:27.736 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:27.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:27.791 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:27.862 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:27.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:27.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:27.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:28.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:28.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:28.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:28.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:28.229 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:27 smithi042 ceph-mon[113926]: pgmap v603: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:28.230 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:27 smithi042 ceph-mon[108570]: pgmap v603: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:27 smithi159 ceph-mon[102519]: pgmap v603: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-13T14:45:28.307 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.350 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:28.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:28.452 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.492 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:28.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:28.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:28.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:28.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.781 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:28.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:28.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:28.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:28.921 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:28.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:29.006 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:45:29.010 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:45:29.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:29.075 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:29.177 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:29.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:29.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2007033008' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3984257482' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:29.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2007033008' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3984257482' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2007033008' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3984257482' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:45:29.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:29.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:29.491 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:29.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:29.629 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:29.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:29.730 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:29.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.771 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:29.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:29.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:29.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:29.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:29.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:29.989 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:29] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.51.0" 2024-09-13T14:45:30.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:30.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:30.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:30.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:30.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:30.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: pgmap v604: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:45:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:45:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:45:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:29 smithi159 ceph-mon[102519]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:45:30.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:30.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: pgmap v604: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:45:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[108570]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: pgmap v604: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 995 op/s 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:45:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:29 smithi042 ceph-mon[113926]: osdmap e173: 8 total, 8 up, 8 in 2024-09-13T14:45:30.445 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:30.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:30.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:30.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:30.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:30.736 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:30.775 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:30.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:30.877 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:30.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:30.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:30.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:31.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:31.041 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:31.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:31.143 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:31.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:31.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:31.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:31.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:31.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:31.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:31.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:31.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:31.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:31.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:31.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:31.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:31.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:31.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:31.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:31.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:31.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:32.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:32.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:32.184 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:32.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:31 smithi159 ceph-mon[102519]: pgmap v606: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1673625039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:32.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:32.321 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:45:32.321 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:45:32.321 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:45:32.321 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:45:32.322 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:45:32.323 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:45:32.323 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:32.323 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:32.323 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:32.323 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:32 smithi042 ceph-mon[113926]: pgmap v606: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:32.323 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1673625039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:32.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:31 smithi042 ceph-mon[108570]: pgmap v606: 99 pgs: 99 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T14:45:32.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1673625039' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:32.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:32.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:32.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:32.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:32.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:32.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:32.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:32.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:32.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:32.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:32.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:32.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:32.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:32.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:33.015 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:33.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:33.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:33.261 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:33.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:33.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:33.440 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:33.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:33.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:33.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:33.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:33.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:33.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:33.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:34.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:34.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:34.213 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:34.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:34 smithi042 ceph-mon[113926]: pgmap v607: 99 pgs: 99 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:34 smithi042 ceph-mon[108570]: pgmap v607: 99 pgs: 99 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:34.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:34.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:34.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:34.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:34 smithi159 ceph-mon[102519]: pgmap v607: 99 pgs: 99 active+clean; 1.9 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:34.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:34.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:34.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:34.651 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:34.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:34.786 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:34.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:34.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:34.931 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:34.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:35.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:35.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.213 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:35.441 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:35.441 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:35.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:35.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:35.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.609 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:35.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:35.750 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:35.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:35.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:35.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:35.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:35.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:35.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:36.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:36.165 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:36.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:36.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:45:36.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:36.306 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:36.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:36.333 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[108570]: pgmap v608: 99 pgs: 99 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-09-13T14:45:36.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:36.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[113926]: pgmap v608: 99 pgs: 99 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-09-13T14:45:36.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:36.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.408 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:36.458 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:36.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:36.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:36 smithi159 ceph-mon[102519]: pgmap v608: 99 pgs: 99 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 968 op/s 2024-09-13T14:45:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:36.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:36.701 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:45:36.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:36.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:36.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:36.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:36.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:36.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:36.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:36.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:37.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:37.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:37.128 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:37.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:37.260 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:37.303 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:37.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:37.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.442 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:37.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:37.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:37.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:37.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:37.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:37.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:37.807 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:37.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:37.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:37.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:37.971 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:37.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:38.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:38.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.172 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:38.212 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:38.245 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:38.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:38.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:38.311 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:38 smithi042 ceph-mon[108570]: pgmap v609: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:45:38.314 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:38 smithi042 ceph-mon[113926]: pgmap v609: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:45:38.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:38.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:38 smithi159 ceph-mon[102519]: pgmap v609: 99 pgs: 99 active+clean; 1.6 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-09-13T14:45:38.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:38.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:38.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:38.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:38.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:38.836 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:38.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:38.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:38.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:39.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:39.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:39.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:39.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:45:39.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:39.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:39.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:39.407 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:39.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:39.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:39.565 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:39.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:39.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:39.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:39.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:39.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:39.888 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:39.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:39.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:40.019 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:39] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-09-13T14:45:40.028 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:40.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:40.130 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:40.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:45:40.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.174 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:40.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:40.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:40.313 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:40 smithi042 ceph-mon[113926]: pgmap v610: 99 pgs: 99 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:40.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:40 smithi042 ceph-mon[108570]: pgmap v610: 99 pgs: 99 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:40.335 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:40.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:40.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:40 smithi159 ceph-mon[102519]: pgmap v610: 99 pgs: 99 active+clean; 1.6 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 964 op/s 2024-09-13T14:45:40.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:40.611 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:40.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:40.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:40.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:40.849 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:40.890 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:40.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:40.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:40.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:41.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:41.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:41.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:41.236 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:41.236 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:41.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:41.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:41.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:41.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:41.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:45:41.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:41.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:41.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.749 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:41.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:41.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:41.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:41.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:41.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:42.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:42 smithi042 ceph-mon[108570]: pgmap v611: 99 pgs: 99 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:42 smithi042 ceph-mon[113926]: pgmap v611: 99 pgs: 99 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:42.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:42.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:42.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:42.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:42.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:42.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:42.407 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:42 smithi159 ceph-mon[102519]: pgmap v611: 99 pgs: 99 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-13T14:45:42.547 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:45:42.547 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:239641 9.2 e173/139 9:47286ecb:::benchmark_data_smithi042_157727_object109692:head v 173'14888, mlcod=173'14885)", 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:41.438104+0000", 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.004288231, 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043135589999999998, 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438104+0000", 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438104+0000", 2024-09-13T14:45:42.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:45:42.550 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438105+0000", 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.23e-07 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438108+0000", 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8460000000000001e-06 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438114+0000", 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1379999999999998e-06 2024-09-13T14:45:42.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438118+0000", 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7899999999999999e-06 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438339+0000", 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022039 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438357+0000", 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8692e-05 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:239650 9.2 e173/139 9:4c5b00dd:::benchmark_data_smithi042_157727_object109701:head v 173'14889, mlcod=173'14885)", 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:41.438150+0000", 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0042430319999999999, 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043290150000000003, 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:45:42.553 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438150+0000", 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438150+0000", 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438150+0000", 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.91e-07 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438156+0000", 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.02e-06 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:45:42.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438161+0000", 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3229999999999997e-06 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438165+0000", 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8850000000000001e-06 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442247+0000", 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040820659999999996 2024-09-13T14:45:42.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442267+0000", 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0211999999999999e-05 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:239652 9.f e173/139 9:f7a41f1c:::benchmark_data_smithi042_157727_object109703:head v 173'14976, mlcod=173'14974)", 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:41.438226+0000", 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0041668340000000003, 2024-09-13T14:45:42.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043017039999999999, 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438226+0000", 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:45:42.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438226+0000", 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438226+0000", 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7899999999999999e-07 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438230+0000", 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3809999999999999e-06 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438237+0000", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2960000000000001e-06 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.438241+0000", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5279999999999997e-06 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.439443+0000", 2024-09-13T14:45:42.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0012020729999999999 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.439465+0000", 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1180000000000001e-05 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:239654 9.a e173/139)", 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:41.442449+0000", 2024-09-13T14:45:42.561 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9999433, 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012250999999999999, 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442449+0000", 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442449+0000", 2024-09-13T14:45:42.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442450+0000", 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2900000000000003e-07 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442455+0000", 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3430000000000003e-06 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:45:42.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442464+0000", 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0310000000000004e-06 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442471+0000", 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2160000000000003e-06 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.564 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:239644 9.d 9:b5541572:::benchmark_data_smithi042_157727_object109695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:41.425767+0000", 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.016625239999999999, 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016849708000000001, 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 239644 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425767+0000", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425767+0000", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425768+0000", 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2799999999999995e-07 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425772+0000", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0319999999999997e-06 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425785+0000", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2880999999999999e-05 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.425791+0000", 2024-09-13T14:45:42.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2580000000000001e-06 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.430186+0000", 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043946580000000001 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.430292+0000", 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010546400000000001 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:45:42.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.430398+0000", 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010609500000000001 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.433989+0000", 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035913429999999999 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:45:42.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:41.442536+0000", 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0085468880000000007 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 5 2024-09-13T14:45:42.570 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:45:42.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:42.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:42.726 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:42.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:42.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:42.783 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:45:42.784 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:45:42.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:42.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:42.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.020 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:43.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:43.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:43.198 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:43.300 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:43.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:43.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:43.481 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:43.516 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:45:43.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:43.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:43.702 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:43.760 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.764 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:43.864 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:43.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:43.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:43.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:43.937 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":612,"stamp":"2024-09-13T14:45:42.258550+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1415780349,"num_objects":21621,"num_object_clones":0,"num_object_copies":64863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21621,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":238363,"num_write_kb":8316578,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125856,"ondisk_log_size":125856,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7808344,"kb_used_data":4209304,"kb_used_omap":28,"kb_used_meta":3598499,"kb_avail":742152872,"statfs":{"total":767960285184,"available":759964540928,"internally_reserved":0,"allocated":4310327296,"data_stored":4303507882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":3684863377},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,4,9,5,7,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-831520768,"num_objects":-12688,"num_object_clones":0,"num_object_copies":-38064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-12688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12688,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.019765"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.016402+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:45:29.016402+0000","last_peered":"2024-09-13T14:45:29.016402+0000","last_clean":"2024-09-13T14:45:29.016402+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:45:29.016402+0000","last_undegraded":"2024-09-13T14:45:29.016402+0000","last_fullsized":"2024-09-13T14:45:29.016402+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.015456+0000","last_change":"2024-09-13T14:43:35.907052+0000","last_active":"2024-09-13T14:45:29.015456+0000","last_peered":"2024-09-13T14:45:29.015456+0000","last_clean":"2024-09-13T14:45:29.015456+0000","last_became_active":"2024-09-13T14:43:35.906588+0000","last_became_peered":"2024-09-13T14:43:35.906588+0000","last_unstale":"2024-09-13T14:45:29.015456+0000","last_undegraded":"2024-09-13T14:45:29.015456+0000","last_fullsized":"2024-09-13T14:45:29.015456+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:56.521107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.950204+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:45:17.950204+0000","last_peered":"2024-09-13T14:45:17.950204+0000","last_clean":"2024-09-13T14:45:17.950204+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:45:17.950204+0000","last_undegraded":"2024-09-13T14:45:17.950204+0000","last_fullsized":"2024-09-13T14:45:17.950204+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":194,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021905+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:45:29.021905+0000","last_peered":"2024-09-13T14:45:29.021905+0000","last_clean":"2024-09-13T14:45:29.021905+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:45:29.021905+0000","last_undegraded":"2024-09-13T14:45:29.021905+0000","last_fullsized":"2024-09-13T14:45:29.021905+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":195,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022324+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:45:29.022324+0000","last_peered":"2024-09-13T14:45:29.022324+0000","last_clean":"2024-09-13T14:45:29.022324+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:45:29.022324+0000","last_undegraded":"2024-09-13T14:45:29.022324+0000","last_fullsized":"2024-09-13T14:45:29.022324+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":28,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945651+0000","last_change":"2024-09-13T14:43:35.907589+0000","last_active":"2024-09-13T14:45:17.945651+0000","last_peered":"2024-09-13T14:45:17.945651+0000","last_clean":"2024-09-13T14:45:17.945651+0000","last_became_active":"2024-09-13T14:43:35.907254+0000","last_became_peered":"2024-09-13T14:43:35.907254+0000","last_unstale":"2024-09-13T14:45:17.945651+0000","last_undegraded":"2024-09-13T14:45:17.945651+0000","last_fullsized":"2024-09-13T14:45:17.945651+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:13.369649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":192,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.013041+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:45:29.013041+0000","last_peered":"2024-09-13T14:45:29.013041+0000","last_clean":"2024-09-13T14:45:29.013041+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:45:29.013041+0000","last_undegraded":"2024-09-13T14:45:29.013041+0000","last_fullsized":"2024-09-13T14:45:29.013041+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":147,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946734+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:45:17.946734+0000","last_peered":"2024-09-13T14:45:17.946734+0000","last_clean":"2024-09-13T14:45:17.946734+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:45:17.946734+0000","last_undegraded":"2024-09-13T14:45:17.946734+0000","last_fullsized":"2024-09-13T14:45:17.946734+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"173'14798","reported_seq":14878,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:38.661120+0000","last_change":"2024-09-13T14:42:07.783733+0000","last_active":"2024-09-13T14:45:38.661120+0000","last_peered":"2024-09-13T14:45:38.661120+0000","last_clean":"2024-09-13T14:45:38.661120+0000","last_became_active":"2024-09-13T14:42:07.783466+0000","last_became_peered":"2024-09-13T14:42:07.783466+0000","last_unstale":"2024-09-13T14:45:38.661120+0000","last_undegraded":"2024-09-13T14:45:38.661120+0000","last_fullsized":"2024-09-13T14:45:38.661120+0000","mapping_epoch":139,"log_start":"153'5700","ondisk_log_start":"153'5700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:20.205570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92930048,"num_objects":1418,"num_object_clones":0,"num_object_copies":4254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14798,"num_write_kb":518912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":40,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946558+0000","last_change":"2024-09-13T14:43:35.907886+0000","last_active":"2024-09-13T14:45:17.946558+0000","last_peered":"2024-09-13T14:45:17.946558+0000","last_clean":"2024-09-13T14:45:17.946558+0000","last_became_active":"2024-09-13T14:43:35.906651+0000","last_became_peered":"2024-09-13T14:43:35.906651+0000","last_unstale":"2024-09-13T14:45:17.946558+0000","last_undegraded":"2024-09-13T14:45:17.946558+0000","last_fullsized":"2024-09-13T14:45:17.946558+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:36.982080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":123,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947120+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:45:17.947120+0000","last_peered":"2024-09-13T14:45:17.947120+0000","last_clean":"2024-09-13T14:45:17.947120+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:45:17.947120+0000","last_undegraded":"2024-09-13T14:45:17.947120+0000","last_fullsized":"2024-09-13T14:45:17.947120+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"173'14626","reported_seq":14707,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:37.994378+0000","last_change":"2024-09-13T14:42:07.780479+0000","last_active":"2024-09-13T14:45:37.994378+0000","last_peered":"2024-09-13T14:45:37.994378+0000","last_clean":"2024-09-13T14:45:37.994378+0000","last_became_active":"2024-09-13T14:42:07.780123+0000","last_became_peered":"2024-09-13T14:42:07.780123+0000","last_unstale":"2024-09-13T14:45:37.994378+0000","last_undegraded":"2024-09-13T14:45:37.994378+0000","last_fullsized":"2024-09-13T14:45:37.994378+0000","mapping_epoch":139,"log_start":"157'7600","ondisk_log_start":"157'7600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:11.339751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92143616,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14626,"num_write_kb":513024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":40,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946732+0000","last_change":"2024-09-13T14:43:35.905294+0000","last_active":"2024-09-13T14:45:17.946732+0000","last_peered":"2024-09-13T14:45:17.946732+0000","last_clean":"2024-09-13T14:45:17.946732+0000","last_became_active":"2024-09-13T14:43:35.904851+0000","last_became_peered":"2024-09-13T14:43:35.904851+0000","last_unstale":"2024-09-13T14:45:17.946732+0000","last_undegraded":"2024-09-13T14:45:17.946732+0000","last_fullsized":"2024-09-13T14:45:17.946732+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:21.591225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945735+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:45:17.945735+0000","last_peered":"2024-09-13T14:45:17.945735+0000","last_clean":"2024-09-13T14:45:17.945735+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:45:17.945735+0000","last_undegraded":"2024-09-13T14:45:17.945735+0000","last_fullsized":"2024-09-13T14:45:17.945735+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"173'14936","reported_seq":14979,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.076359+0000","last_change":"2024-09-13T14:42:07.781173+0000","last_active":"2024-09-13T14:45:41.076359+0000","last_peered":"2024-09-13T14:45:41.076359+0000","last_clean":"2024-09-13T14:45:41.076359+0000","last_became_active":"2024-09-13T14:42:07.780794+0000","last_became_peered":"2024-09-13T14:42:07.780794+0000","last_unstale":"2024-09-13T14:45:41.076359+0000","last_undegraded":"2024-09-13T14:45:41.076359+0000","last_fullsized":"2024-09-13T14:45:41.076359+0000","mapping_epoch":139,"log_start":"153'6100","ondisk_log_start":"153'6100","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8836,"log_dups_size":0,"ondisk_log_size":8836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:03.397389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83755008,"num_objects":1278,"num_object_clones":0,"num_object_copies":3834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14936,"num_write_kb":518848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":40,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.950167+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:45:17.950167+0000","last_peered":"2024-09-13T14:45:17.950167+0000","last_clean":"2024-09-13T14:45:17.950167+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:45:17.950167+0000","last_undegraded":"2024-09-13T14:45:17.950167+0000","last_fullsized":"2024-09-13T14:45:17.950167+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":139,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.943162+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:45:17.943162+0000","last_peered":"2024-09-13T14:45:17.943162+0000","last_clean":"2024-09-13T14:45:17.943162+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:45:17.943162+0000","last_undegraded":"2024-09-13T14:45:17.943162+0000","last_fullsized":"2024-09-13T14:45:17.943162+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"173'14646","reported_seq":14690,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.070878+0000","last_change":"2024-09-13T14:42:07.781817+0000","last_active":"2024-09-13T14:45:41.070878+0000","last_peered":"2024-09-13T14:45:41.070878+0000","last_clean":"2024-09-13T14:45:41.070878+0000","last_became_active":"2024-09-13T14:42:07.781534+0000","last_became_peered":"2024-09-13T14:42:07.781534+0000","last_unstale":"2024-09-13T14:45:41.070878+0000","last_undegraded":"2024-09-13T14:45:41.070878+0000","last_fullsized":"2024-09-13T14:45:41.070878+0000","mapping_epoch":139,"log_start":"153'5800","ondisk_log_start":"153'5800","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8846,"log_dups_size":0,"ondisk_log_size":8846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:27.372521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82051072,"num_objects":1252,"num_object_clones":0,"num_object_copies":3756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14646,"num_write_kb":508736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017948+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:45:29.017948+0000","last_peered":"2024-09-13T14:45:29.017948+0000","last_clean":"2024-09-13T14:45:29.017948+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:45:29.017948+0000","last_undegraded":"2024-09-13T14:45:29.017948+0000","last_fullsized":"2024-09-13T14:45:29.017948+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":214,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022229+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:45:29.022229+0000","last_peered":"2024-09-13T14:45:29.022229+0000","last_clean":"2024-09-13T14:45:29.022229+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:45:29.022229+0000","last_undegraded":"2024-09-13T14:45:29.022229+0000","last_fullsized":"2024-09-13T14:45:29.022229+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":515,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011422+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:45:29.011422+0000","last_peered":"2024-09-13T14:45:29.011422+0000","last_clean":"2024-09-13T14:45:29.011422+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:45:29.011422+0000","last_undegraded":"2024-09-13T14:45:29.011422+0000","last_fullsized":"2024-09-13T14:45:29.011422+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1","version":"173'14955","reported_seq":15036,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.687770+0000","last_change":"2024-09-13T14:42:07.942573+0000","last_active":"2024-09-13T14:45:40.687770+0000","last_peered":"2024-09-13T14:45:40.687770+0000","last_clean":"2024-09-13T14:45:40.687770+0000","last_became_active":"2024-09-13T14:42:07.942229+0000","last_became_peered":"2024-09-13T14:42:07.942229+0000","last_unstale":"2024-09-13T14:45:40.687770+0000","last_undegraded":"2024-09-13T14:45:40.687770+0000","last_fullsized":"2024-09-13T14:45:40.687770+0000","mapping_epoch":139,"log_start":"158'8700","ondisk_log_start":"158'8700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:10.238039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14955,"num_write_kb":520576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011167+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:45:29.011167+0000","last_peered":"2024-09-13T14:45:29.011167+0000","last_clean":"2024-09-13T14:45:29.011167+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:45:29.011167+0000","last_undegraded":"2024-09-13T14:45:29.011167+0000","last_fullsized":"2024-09-13T14:45:29.011167+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019664+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:45:29.019664+0000","last_peered":"2024-09-13T14:45:29.019664+0000","last_clean":"2024-09-13T14:45:29.019664+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:45:29.019664+0000","last_undegraded":"2024-09-13T14:45:29.019664+0000","last_fullsized":"2024-09-13T14:45:29.019664+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":408,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946971+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:45:17.946971+0000","last_peered":"2024-09-13T14:45:17.946971+0000","last_clean":"2024-09-13T14:45:17.946971+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:45:17.946971+0000","last_undegraded":"2024-09-13T14:45:17.946971+0000","last_fullsized":"2024-09-13T14:45:17.946971+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.0","version":"173'14709","reported_seq":14790,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:38.026930+0000","last_change":"2024-09-13T14:42:07.942557+0000","last_active":"2024-09-13T14:45:38.026930+0000","last_peered":"2024-09-13T14:45:38.026930+0000","last_clean":"2024-09-13T14:45:38.026930+0000","last_became_active":"2024-09-13T14:42:07.942184+0000","last_became_peered":"2024-09-13T14:42:07.942184+0000","last_unstale":"2024-09-13T14:45:38.026930+0000","last_undegraded":"2024-09-13T14:45:38.026930+0000","last_fullsized":"2024-09-13T14:45:38.026930+0000","mapping_epoch":139,"log_start":"157'7700","ondisk_log_start":"157'7700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":7009,"log_dups_size":0,"ondisk_log_size":7009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:04.365348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14709,"num_write_kb":518656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.943298+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:45:17.943298+0000","last_peered":"2024-09-13T14:45:17.943298+0000","last_clean":"2024-09-13T14:45:17.943298+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:45:17.943298+0000","last_undegraded":"2024-09-13T14:45:17.943298+0000","last_fullsized":"2024-09-13T14:45:17.943298+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019608+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:45:29.019608+0000","last_peered":"2024-09-13T14:45:29.019608+0000","last_clean":"2024-09-13T14:45:29.019608+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:45:29.019608+0000","last_undegraded":"2024-09-13T14:45:29.019608+0000","last_fullsized":"2024-09-13T14:45:29.019608+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"173'15321","reported_seq":15389,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.656851+0000","last_change":"2024-09-13T14:43:35.927511+0000","last_active":"2024-09-13T14:45:40.656851+0000","last_peered":"2024-09-13T14:45:40.656851+0000","last_clean":"2024-09-13T14:45:40.656851+0000","last_became_active":"2024-09-13T14:43:35.907655+0000","last_became_peered":"2024-09-13T14:43:35.907655+0000","last_unstale":"2024-09-13T14:45:40.656851+0000","last_undegraded":"2024-09-13T14:45:40.656851+0000","last_fullsized":"2024-09-13T14:45:40.656851+0000","mapping_epoch":156,"log_start":"158'9044","ondisk_log_start":"158'9044","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6277,"log_dups_size":0,"ondisk_log_size":6277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:43.071844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15321,"num_write_kb":531776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":140,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945803+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:45:17.945803+0000","last_peered":"2024-09-13T14:45:17.945803+0000","last_clean":"2024-09-13T14:45:17.945803+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:45:17.945803+0000","last_undegraded":"2024-09-13T14:45:17.945803+0000","last_fullsized":"2024-09-13T14:45:17.945803+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":148,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.028996+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:45:29.028996+0000","last_peered":"2024-09-13T14:45:29.028996+0000","last_clean":"2024-09-13T14:45:29.028996+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:45:29.028996+0000","last_undegraded":"2024-09-13T14:45:29.028996+0000","last_fullsized":"2024-09-13T14:45:29.028996+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":408,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947027+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:45:17.947027+0000","last_peered":"2024-09-13T14:45:17.947027+0000","last_clean":"2024-09-13T14:45:17.947027+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:45:17.947027+0000","last_undegraded":"2024-09-13T14:45:17.947027+0000","last_fullsized":"2024-09-13T14:45:17.947027+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"173'14858","reported_seq":14939,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.659430+0000","last_change":"2024-09-13T14:42:07.780605+0000","last_active":"2024-09-13T14:45:40.659430+0000","last_peered":"2024-09-13T14:45:40.659430+0000","last_clean":"2024-09-13T14:45:40.659430+0000","last_became_active":"2024-09-13T14:42:07.780309+0000","last_became_peered":"2024-09-13T14:42:07.780309+0000","last_unstale":"2024-09-13T14:45:40.659430+0000","last_undegraded":"2024-09-13T14:45:40.659430+0000","last_fullsized":"2024-09-13T14:45:40.659430+0000","mapping_epoch":139,"log_start":"158'8600","ondisk_log_start":"158'8600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:17.130563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14858,"num_write_kb":516864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":168,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011291+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:45:29.011291+0000","last_peered":"2024-09-13T14:45:29.011291+0000","last_clean":"2024-09-13T14:45:29.011291+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:45:29.011291+0000","last_undegraded":"2024-09-13T14:45:29.011291+0000","last_fullsized":"2024-09-13T14:45:29.011291+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.020975+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:45:29.020975+0000","last_peered":"2024-09-13T14:45:29.020975+0000","last_clean":"2024-09-13T14:45:29.020975+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:45:29.020975+0000","last_undegraded":"2024-09-13T14:45:29.020975+0000","last_fullsized":"2024-09-13T14:45:29.020975+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"173'15030","reported_seq":15111,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.655005+0000","last_change":"2024-09-13T14:42:07.782112+0000","last_active":"2024-09-13T14:45:40.655005+0000","last_peered":"2024-09-13T14:45:40.655005+0000","last_clean":"2024-09-13T14:45:40.655005+0000","last_became_active":"2024-09-13T14:42:07.781780+0000","last_became_peered":"2024-09-13T14:42:07.781780+0000","last_unstale":"2024-09-13T14:45:40.655005+0000","last_undegraded":"2024-09-13T14:45:40.655005+0000","last_fullsized":"2024-09-13T14:45:40.655005+0000","mapping_epoch":139,"log_start":"158'8700","ondisk_log_start":"158'8700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6330,"log_dups_size":0,"ondisk_log_size":6330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:55.236172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87818240,"num_objects":1340,"num_object_clones":0,"num_object_copies":4020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15030,"num_write_kb":523840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":413,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021741+0000","last_change":"2024-09-13T14:43:37.621179+0000","last_active":"2024-09-13T14:45:29.021741+0000","last_peered":"2024-09-13T14:45:29.021741+0000","last_clean":"2024-09-13T14:45:29.021741+0000","last_became_active":"2024-09-13T14:43:35.909308+0000","last_became_peered":"2024-09-13T14:43:35.909308+0000","last_unstale":"2024-09-13T14:45:29.021741+0000","last_undegraded":"2024-09-13T14:45:29.021741+0000","last_fullsized":"2024-09-13T14:45:29.021741+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:37.621104+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:37.621104+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.210888+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":153,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947330+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:45:17.947330+0000","last_peered":"2024-09-13T14:45:17.947330+0000","last_clean":"2024-09-13T14:45:17.947330+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:45:17.947330+0000","last_undegraded":"2024-09-13T14:45:17.947330+0000","last_fullsized":"2024-09-13T14:45:17.947330+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011513+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:45:29.011513+0000","last_peered":"2024-09-13T14:45:29.011513+0000","last_clean":"2024-09-13T14:45:29.011513+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:45:29.011513+0000","last_undegraded":"2024-09-13T14:45:29.011513+0000","last_fullsized":"2024-09-13T14:45:29.011513+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":124,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.943463+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:45:17.943463+0000","last_peered":"2024-09-13T14:45:17.943463+0000","last_clean":"2024-09-13T14:45:17.943463+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:45:17.943463+0000","last_undegraded":"2024-09-13T14:45:17.943463+0000","last_fullsized":"2024-09-13T14:45:17.943463+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":426,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.015283+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:45:29.015283+0000","last_peered":"2024-09-13T14:45:29.015283+0000","last_clean":"2024-09-13T14:45:29.015283+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:45:29.015283+0000","last_undegraded":"2024-09-13T14:45:29.015283+0000","last_fullsized":"2024-09-13T14:45:29.015283+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.b","version":"173'14915","reported_seq":14996,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:38.660878+0000","last_change":"2024-09-13T14:42:07.942512+0000","last_active":"2024-09-13T14:45:38.660878+0000","last_peered":"2024-09-13T14:45:38.660878+0000","last_clean":"2024-09-13T14:45:38.660878+0000","last_became_active":"2024-09-13T14:42:07.942191+0000","last_became_peered":"2024-09-13T14:42:07.942191+0000","last_unstale":"2024-09-13T14:45:38.660878+0000","last_undegraded":"2024-09-13T14:45:38.660878+0000","last_fullsized":"2024-09-13T14:45:38.660878+0000","mapping_epoch":139,"log_start":"153'5800","ondisk_log_start":"153'5800","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9115,"log_dups_size":0,"ondisk_log_size":9115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:56.446026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93126656,"num_objects":1421,"num_object_clones":0,"num_object_copies":4263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14915,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019846+0000","last_change":"2024-09-13T14:43:35.907156+0000","last_active":"2024-09-13T14:45:29.019846+0000","last_peered":"2024-09-13T14:45:29.019846+0000","last_clean":"2024-09-13T14:45:29.019846+0000","last_became_active":"2024-09-13T14:43:35.906893+0000","last_became_peered":"2024-09-13T14:43:35.906893+0000","last_unstale":"2024-09-13T14:45:29.019846+0000","last_undegraded":"2024-09-13T14:45:29.019846+0000","last_fullsized":"2024-09-13T14:45:29.019846+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:36.233310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":194,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022839+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:45:29.022839+0000","last_peered":"2024-09-13T14:45:29.022839+0000","last_clean":"2024-09-13T14:45:29.022839+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:45:29.022839+0000","last_undegraded":"2024-09-13T14:45:29.022839+0000","last_fullsized":"2024-09-13T14:45:29.022839+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":207,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022191+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:45:29.022191+0000","last_peered":"2024-09-13T14:45:29.022191+0000","last_clean":"2024-09-13T14:45:29.022191+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:45:29.022191+0000","last_undegraded":"2024-09-13T14:45:29.022191+0000","last_fullsized":"2024-09-13T14:45:29.022191+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":412,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021734+0000","last_change":"2024-09-13T14:43:36.606414+0000","last_active":"2024-09-13T14:45:29.021734+0000","last_peered":"2024-09-13T14:45:29.021734+0000","last_clean":"2024-09-13T14:45:29.021734+0000","last_became_active":"2024-09-13T14:43:35.907350+0000","last_became_peered":"2024-09-13T14:43:35.907350+0000","last_unstale":"2024-09-13T14:45:29.021734+0000","last_undegraded":"2024-09-13T14:45:29.021734+0000","last_fullsized":"2024-09-13T14:45:29.021734+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:36.606328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:36.606328+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:34.516165+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"173'15006","reported_seq":15050,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.088336+0000","last_change":"2024-09-13T14:42:07.781099+0000","last_active":"2024-09-13T14:45:41.088336+0000","last_peered":"2024-09-13T14:45:41.088336+0000","last_clean":"2024-09-13T14:45:41.088336+0000","last_became_active":"2024-09-13T14:42:07.780786+0000","last_became_peered":"2024-09-13T14:42:07.780786+0000","last_unstale":"2024-09-13T14:45:41.088336+0000","last_undegraded":"2024-09-13T14:45:41.088336+0000","last_fullsized":"2024-09-13T14:45:41.088336+0000","mapping_epoch":139,"log_start":"153'6100","ondisk_log_start":"153'6100","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8906,"log_dups_size":0,"ondisk_log_size":8906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:24.082388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88080384,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15006,"num_write_kb":523200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":190,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.007010+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:45:29.007010+0000","last_peered":"2024-09-13T14:45:29.007010+0000","last_clean":"2024-09-13T14:45:29.007010+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:45:29.007010+0000","last_undegraded":"2024-09-13T14:45:29.007010+0000","last_fullsized":"2024-09-13T14:45:29.007010+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":164,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019936+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:45:29.019936+0000","last_peered":"2024-09-13T14:45:29.019936+0000","last_clean":"2024-09-13T14:45:29.019936+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:45:29.019936+0000","last_undegraded":"2024-09-13T14:45:29.019936+0000","last_fullsized":"2024-09-13T14:45:29.019936+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":201,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.020457+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:45:29.020457+0000","last_peered":"2024-09-13T14:45:29.020457+0000","last_clean":"2024-09-13T14:45:29.020457+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:45:29.020457+0000","last_undegraded":"2024-09-13T14:45:29.020457+0000","last_fullsized":"2024-09-13T14:45:29.020457+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":408,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947028+0000","last_change":"2024-09-13T14:42:56.295979+0000","last_active":"2024-09-13T14:45:17.947028+0000","last_peered":"2024-09-13T14:45:17.947028+0000","last_clean":"2024-09-13T14:45:17.947028+0000","last_became_active":"2024-09-13T14:42:56.290769+0000","last_became_peered":"2024-09-13T14:42:56.290769+0000","last_unstale":"2024-09-13T14:45:17.947028+0000","last_undegraded":"2024-09-13T14:45:17.947028+0000","last_fullsized":"2024-09-13T14:45:17.947028+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:27.386430+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"173'14730","reported_seq":14811,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:38.663212+0000","last_change":"2024-09-13T14:42:07.783147+0000","last_active":"2024-09-13T14:45:38.663212+0000","last_peered":"2024-09-13T14:45:38.663212+0000","last_clean":"2024-09-13T14:45:38.663212+0000","last_became_active":"2024-09-13T14:42:07.782831+0000","last_became_peered":"2024-09-13T14:42:07.782831+0000","last_unstale":"2024-09-13T14:45:38.663212+0000","last_undegraded":"2024-09-13T14:45:38.663212+0000","last_fullsized":"2024-09-13T14:45:38.663212+0000","mapping_epoch":139,"log_start":"153'5600","ondisk_log_start":"153'5600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9130,"log_dups_size":0,"ondisk_log_size":9130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:23.396196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14730,"num_write_kb":514816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":154,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947207+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:45:17.947207+0000","last_peered":"2024-09-13T14:45:17.947207+0000","last_clean":"2024-09-13T14:45:17.947207+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:45:17.947207+0000","last_undegraded":"2024-09-13T14:45:17.947207+0000","last_fullsized":"2024-09-13T14:45:17.947207+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":90,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011021+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:45:29.011021+0000","last_peered":"2024-09-13T14:45:29.011021+0000","last_clean":"2024-09-13T14:45:29.011021+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:45:29.011021+0000","last_undegraded":"2024-09-13T14:45:29.011021+0000","last_fullsized":"2024-09-13T14:45:29.011021+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":177,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.013397+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:45:29.013397+0000","last_peered":"2024-09-13T14:45:29.013397+0000","last_clean":"2024-09-13T14:45:29.013397+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:45:29.013397+0000","last_undegraded":"2024-09-13T14:45:29.013397+0000","last_fullsized":"2024-09-13T14:45:29.013397+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":416,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.943916+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:45:17.943916+0000","last_peered":"2024-09-13T14:45:17.943916+0000","last_clean":"2024-09-13T14:45:17.943916+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:45:17.943916+0000","last_undegraded":"2024-09-13T14:45:17.943916+0000","last_fullsized":"2024-09-13T14:45:17.943916+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.f","version":"173'14957","reported_seq":15000,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.076505+0000","last_change":"2024-09-13T14:42:07.781837+0000","last_active":"2024-09-13T14:45:41.076505+0000","last_peered":"2024-09-13T14:45:41.076505+0000","last_clean":"2024-09-13T14:45:41.076505+0000","last_became_active":"2024-09-13T14:42:07.781584+0000","last_became_peered":"2024-09-13T14:42:07.781584+0000","last_unstale":"2024-09-13T14:45:41.076505+0000","last_undegraded":"2024-09-13T14:45:41.076505+0000","last_fullsized":"2024-09-13T14:45:41.076505+0000","mapping_epoch":139,"log_start":"153'6100","ondisk_log_start":"153'6100","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8857,"log_dups_size":0,"ondisk_log_size":8857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:21.323962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14957,"num_write_kb":522624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":154,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.950249+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:45:17.950249+0000","last_peered":"2024-09-13T14:45:17.950249+0000","last_clean":"2024-09-13T14:45:17.950249+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:45:17.950249+0000","last_undegraded":"2024-09-13T14:45:17.950249+0000","last_fullsized":"2024-09-13T14:45:17.950249+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":215,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.013587+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:45:29.013587+0000","last_peered":"2024-09-13T14:45:29.013587+0000","last_clean":"2024-09-13T14:45:29.013587+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:45:29.013587+0000","last_undegraded":"2024-09-13T14:45:29.013587+0000","last_fullsized":"2024-09-13T14:45:29.013587+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":86,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.014267+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:45:29.014267+0000","last_peered":"2024-09-13T14:45:29.014267+0000","last_clean":"2024-09-13T14:45:29.014267+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:45:29.014267+0000","last_undegraded":"2024-09-13T14:45:29.014267+0000","last_fullsized":"2024-09-13T14:45:29.014267+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":416,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.943687+0000","last_change":"2024-09-13T14:42:57.907376+0000","last_active":"2024-09-13T14:45:17.943687+0000","last_peered":"2024-09-13T14:45:17.943687+0000","last_clean":"2024-09-13T14:45:17.943687+0000","last_became_active":"2024-09-13T14:42:56.286461+0000","last_became_peered":"2024-09-13T14:42:56.286461+0000","last_unstale":"2024-09-13T14:45:17.943687+0000","last_undegraded":"2024-09-13T14:45:17.943687+0000","last_fullsized":"2024-09-13T14:45:17.943687+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:27.858330+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.e","version":"173'15011","reported_seq":15061,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.189951+0000","last_change":"2024-09-13T14:43:35.917618+0000","last_active":"2024-09-13T14:45:41.189951+0000","last_peered":"2024-09-13T14:45:41.189951+0000","last_clean":"2024-09-13T14:45:41.189951+0000","last_became_active":"2024-09-13T14:43:35.904748+0000","last_became_peered":"2024-09-13T14:43:35.904748+0000","last_unstale":"2024-09-13T14:45:41.189951+0000","last_undegraded":"2024-09-13T14:45:41.189951+0000","last_fullsized":"2024-09-13T14:45:41.189951+0000","mapping_epoch":156,"log_start":"153'6100","ondisk_log_start":"153'6100","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8911,"log_dups_size":0,"ondisk_log_size":8911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:17.729503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15011,"num_write_kb":521856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":164,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.013496+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:45:29.013496+0000","last_peered":"2024-09-13T14:45:29.013496+0000","last_clean":"2024-09-13T14:45:29.013496+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:45:29.013496+0000","last_undegraded":"2024-09-13T14:45:29.013496+0000","last_fullsized":"2024-09-13T14:45:29.013496+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":107,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022982+0000","last_change":"2024-09-13T14:43:35.908853+0000","last_active":"2024-09-13T14:45:29.022982+0000","last_peered":"2024-09-13T14:45:29.022982+0000","last_clean":"2024-09-13T14:45:29.022982+0000","last_became_active":"2024-09-13T14:43:35.908544+0000","last_became_peered":"2024-09-13T14:43:35.908544+0000","last_unstale":"2024-09-13T14:45:29.022982+0000","last_undegraded":"2024-09-13T14:45:29.022982+0000","last_fullsized":"2024-09-13T14:45:29.022982+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:13.786745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":185,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022308+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:45:29.022308+0000","last_peered":"2024-09-13T14:45:29.022308+0000","last_clean":"2024-09-13T14:45:29.022308+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:45:29.022308+0000","last_undegraded":"2024-09-13T14:45:29.022308+0000","last_fullsized":"2024-09-13T14:45:29.022308+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":408,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946532+0000","last_change":"2024-09-13T14:43:00.868011+0000","last_active":"2024-09-13T14:45:17.946532+0000","last_peered":"2024-09-13T14:45:17.946532+0000","last_clean":"2024-09-13T14:45:17.946532+0000","last_became_active":"2024-09-13T14:42:56.283775+0000","last_became_peered":"2024-09-13T14:42:56.283775+0000","last_unstale":"2024-09-13T14:45:17.946532+0000","last_undegraded":"2024-09-13T14:45:17.946532+0000","last_fullsized":"2024-09-13T14:45:17.946532+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:00.867870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:00.867870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:25.806175+0000","scrub_duration":0.0043830680000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.d","version":"173'14785","reported_seq":14854,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:38.002766+0000","last_change":"2024-09-13T14:43:35.914172+0000","last_active":"2024-09-13T14:45:38.002766+0000","last_peered":"2024-09-13T14:45:38.002766+0000","last_clean":"2024-09-13T14:45:38.002766+0000","last_became_active":"2024-09-13T14:43:35.905309+0000","last_became_peered":"2024-09-13T14:43:35.905309+0000","last_unstale":"2024-09-13T14:45:38.002766+0000","last_undegraded":"2024-09-13T14:45:38.002766+0000","last_fullsized":"2024-09-13T14:45:38.002766+0000","mapping_epoch":156,"log_start":"157'7797","ondisk_log_start":"157'7797","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:30.311540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93978648,"num_objects":1435,"num_object_clones":0,"num_object_copies":4305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1435,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14785,"num_write_kb":518977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.013270+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:45:29.013270+0000","last_peered":"2024-09-13T14:45:29.013270+0000","last_clean":"2024-09-13T14:45:29.013270+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:45:29.013270+0000","last_undegraded":"2024-09-13T14:45:29.013270+0000","last_fullsized":"2024-09-13T14:45:29.013270+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":165,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017524+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:45:29.017524+0000","last_peered":"2024-09-13T14:45:29.017524+0000","last_clean":"2024-09-13T14:45:29.017524+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:45:29.017524+0000","last_undegraded":"2024-09-13T14:45:29.017524+0000","last_fullsized":"2024-09-13T14:45:29.017524+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":151,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.010829+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:45:29.010829+0000","last_peered":"2024-09-13T14:45:29.010829+0000","last_clean":"2024-09-13T14:45:29.010829+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:45:29.010829+0000","last_undegraded":"2024-09-13T14:45:29.010829+0000","last_fullsized":"2024-09-13T14:45:29.010829+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":426,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.012283+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:45:29.012283+0000","last_peered":"2024-09-13T14:45:29.012283+0000","last_clean":"2024-09-13T14:45:29.012283+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:45:29.012283+0000","last_undegraded":"2024-09-13T14:45:29.012283+0000","last_fullsized":"2024-09-13T14:45:29.012283+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"173'15016","reported_seq":15097,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.703481+0000","last_change":"2024-09-13T14:42:07.782967+0000","last_active":"2024-09-13T14:45:40.703481+0000","last_peered":"2024-09-13T14:45:40.703481+0000","last_clean":"2024-09-13T14:45:40.703481+0000","last_became_active":"2024-09-13T14:42:07.782365+0000","last_became_peered":"2024-09-13T14:42:07.782365+0000","last_unstale":"2024-09-13T14:45:40.703481+0000","last_undegraded":"2024-09-13T14:45:40.703481+0000","last_fullsized":"2024-09-13T14:45:40.703481+0000","mapping_epoch":139,"log_start":"158'8700","ondisk_log_start":"158'8700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:18.000113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83099648,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15016,"num_write_kb":521088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":199,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022449+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:45:29.022449+0000","last_peered":"2024-09-13T14:45:29.022449+0000","last_clean":"2024-09-13T14:45:29.022449+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:45:29.022449+0000","last_undegraded":"2024-09-13T14:45:29.022449+0000","last_fullsized":"2024-09-13T14:45:29.022449+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021555+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:45:29.021555+0000","last_peered":"2024-09-13T14:45:29.021555+0000","last_clean":"2024-09-13T14:45:29.021555+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:45:29.021555+0000","last_undegraded":"2024-09-13T14:45:29.021555+0000","last_fullsized":"2024-09-13T14:45:29.021555+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":125,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019192+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:45:29.019192+0000","last_peered":"2024-09-13T14:45:29.019192+0000","last_clean":"2024-09-13T14:45:29.019192+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:45:29.019192+0000","last_undegraded":"2024-09-13T14:45:29.019192+0000","last_fullsized":"2024-09-13T14:45:29.019192+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946106+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:45:17.946106+0000","last_peered":"2024-09-13T14:45:17.946106+0000","last_clean":"2024-09-13T14:45:17.946106+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:45:17.946106+0000","last_undegraded":"2024-09-13T14:45:17.946106+0000","last_fullsized":"2024-09-13T14:45:17.946106+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":147,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945430+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:45:17.945430+0000","last_peered":"2024-09-13T14:45:17.945430+0000","last_clean":"2024-09-13T14:45:17.945430+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:45:17.945430+0000","last_undegraded":"2024-09-13T14:45:17.945430+0000","last_fullsized":"2024-09-13T14:45:17.945430+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.014422+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:45:29.014422+0000","last_peered":"2024-09-13T14:45:29.014422+0000","last_clean":"2024-09-13T14:45:29.014422+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:45:29.014422+0000","last_undegraded":"2024-09-13T14:45:29.014422+0000","last_fullsized":"2024-09-13T14:45:29.014422+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":111,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.007122+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:45:29.007122+0000","last_peered":"2024-09-13T14:45:29.007122+0000","last_clean":"2024-09-13T14:45:29.007122+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:45:29.007122+0000","last_undegraded":"2024-09-13T14:45:29.007122+0000","last_fullsized":"2024-09-13T14:45:29.007122+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":138,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947213+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:45:17.947213+0000","last_peered":"2024-09-13T14:45:17.947213+0000","last_clean":"2024-09-13T14:45:17.947213+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:45:17.947213+0000","last_undegraded":"2024-09-13T14:45:17.947213+0000","last_fullsized":"2024-09-13T14:45:17.947213+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011626+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:45:29.011626+0000","last_peered":"2024-09-13T14:45:29.011626+0000","last_clean":"2024-09-13T14:45:29.011626+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:45:29.011626+0000","last_undegraded":"2024-09-13T14:45:29.011626+0000","last_fullsized":"2024-09-13T14:45:29.011626+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022696+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:45:29.022696+0000","last_peered":"2024-09-13T14:45:29.022696+0000","last_clean":"2024-09-13T14:45:29.022696+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:45:29.022696+0000","last_undegraded":"2024-09-13T14:45:29.022696+0000","last_fullsized":"2024-09-13T14:45:29.022696+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.010727+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:45:29.010727+0000","last_peered":"2024-09-13T14:45:29.010727+0000","last_clean":"2024-09-13T14:45:29.010727+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:45:29.010727+0000","last_undegraded":"2024-09-13T14:45:29.010727+0000","last_fullsized":"2024-09-13T14:45:29.010727+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":140,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021523+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:45:29.021523+0000","last_peered":"2024-09-13T14:45:29.021523+0000","last_clean":"2024-09-13T14:45:29.021523+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:45:29.021523+0000","last_undegraded":"2024-09-13T14:45:29.021523+0000","last_fullsized":"2024-09-13T14:45:29.021523+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":182,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011019+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:45:29.011019+0000","last_peered":"2024-09-13T14:45:29.011019+0000","last_clean":"2024-09-13T14:45:29.011019+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:45:29.011019+0000","last_undegraded":"2024-09-13T14:45:29.011019+0000","last_fullsized":"2024-09-13T14:45:29.011019+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":196,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022240+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:45:29.022240+0000","last_peered":"2024-09-13T14:45:29.022240+0000","last_clean":"2024-09-13T14:45:29.022240+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:45:29.022240+0000","last_undegraded":"2024-09-13T14:45:29.022240+0000","last_fullsized":"2024-09-13T14:45:29.022240+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.015201+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:45:29.015201+0000","last_peered":"2024-09-13T14:45:29.015201+0000","last_clean":"2024-09-13T14:45:29.015201+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:45:29.015201+0000","last_undegraded":"2024-09-13T14:45:29.015201+0000","last_fullsized":"2024-09-13T14:45:29.015201+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":166,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011349+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:45:29.011349+0000","last_peered":"2024-09-13T14:45:29.011349+0000","last_clean":"2024-09-13T14:45:29.011349+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:45:29.011349+0000","last_undegraded":"2024-09-13T14:45:29.011349+0000","last_fullsized":"2024-09-13T14:45:29.011349+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":90,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011545+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:45:29.011545+0000","last_peered":"2024-09-13T14:45:29.011545+0000","last_clean":"2024-09-13T14:45:29.011545+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:45:29.011545+0000","last_undegraded":"2024-09-13T14:45:29.011545+0000","last_fullsized":"2024-09-13T14:45:29.011545+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":191,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.006927+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:45:29.006927+0000","last_peered":"2024-09-13T14:45:29.006927+0000","last_clean":"2024-09-13T14:45:29.006927+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:45:29.006927+0000","last_undegraded":"2024-09-13T14:45:29.006927+0000","last_fullsized":"2024-09-13T14:45:29.006927+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":199,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022389+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:45:29.022389+0000","last_peered":"2024-09-13T14:45:29.022389+0000","last_clean":"2024-09-13T14:45:29.022389+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:45:29.022389+0000","last_undegraded":"2024-09-13T14:45:29.022389+0000","last_fullsized":"2024-09-13T14:45:29.022389+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":152,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.946039+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:45:17.946039+0000","last_peered":"2024-09-13T14:45:17.946039+0000","last_clean":"2024-09-13T14:45:17.946039+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:45:17.946039+0000","last_undegraded":"2024-09-13T14:45:17.946039+0000","last_fullsized":"2024-09-13T14:45:17.946039+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":139,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.947298+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:45:17.947298+0000","last_peered":"2024-09-13T14:45:17.947298+0000","last_clean":"2024-09-13T14:45:17.947298+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:45:17.947298+0000","last_undegraded":"2024-09-13T14:45:17.947298+0000","last_fullsized":"2024-09-13T14:45:17.947298+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":152,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945993+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:45:17.945993+0000","last_peered":"2024-09-13T14:45:17.945993+0000","last_clean":"2024-09-13T14:45:17.945993+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:45:17.945993+0000","last_undegraded":"2024-09-13T14:45:17.945993+0000","last_fullsized":"2024-09-13T14:45:17.945993+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":124,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.950258+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:45:17.950258+0000","last_peered":"2024-09-13T14:45:17.950258+0000","last_clean":"2024-09-13T14:45:17.950258+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:45:17.950258+0000","last_undegraded":"2024-09-13T14:45:17.950258+0000","last_fullsized":"2024-09-13T14:45:17.950258+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":198,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.012170+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:45:29.012170+0000","last_peered":"2024-09-13T14:45:29.012170+0000","last_clean":"2024-09-13T14:45:29.012170+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:45:29.012170+0000","last_undegraded":"2024-09-13T14:45:29.012170+0000","last_fullsized":"2024-09-13T14:45:29.012170+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":187,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.011120+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:45:29.011120+0000","last_peered":"2024-09-13T14:45:29.011120+0000","last_clean":"2024-09-13T14:45:29.011120+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:45:29.011120+0000","last_undegraded":"2024-09-13T14:45:29.011120+0000","last_fullsized":"2024-09-13T14:45:29.011120+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":172,"state":"active+clean","last_fresh":"2024-09-13T14:45:17.945501+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:45:17.945501+0000","last_peered":"2024-09-13T14:45:17.945501+0000","last_clean":"2024-09-13T14:45:17.945501+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:45:17.945501+0000","last_undegraded":"2024-09-13T14:45:17.945501+0000","last_fullsized":"2024-09-13T14:45:17.945501+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":200,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.022517+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:45:29.022517+0000","last_peered":"2024-09-13T14:45:29.022517+0000","last_clean":"2024-09-13T14:45:29.022517+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:45:29.022517+0000","last_undegraded":"2024-09-13T14:45:29.022517+0000","last_fullsized":"2024-09-13T14:45:29.022517+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1415184408,"num_objects":21595,"num_object_clones":0,"num_object_copies":64785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21595,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":238299,"num_write_kb":8316545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4178194432,"data_stored":4178182216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124158,"ondisk_log_size":124158,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234307,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370520,"kb_used_data":743632,"kb_used_omap":1,"kb_used_meta":626686,"kb_avail":92374632,"statfs":{"total":95995035648,"available":94591623168,"internally_reserved":0,"allocated":761479168,"data_stored":760616136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":641726923},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495945,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":801936,"kb_used_data":257288,"kb_used_omap":1,"kb_used_meta":544638,"kb_avail":92943216,"statfs":{"total":95995035648,"available":95173853184,"internally_reserved":0,"allocated":263462912,"data_stored":262608011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":557709767},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757583,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557900,"kb_used_data":764676,"kb_used_omap":1,"kb_used_meta":793214,"kb_avail":92187252,"statfs":{"total":95995035648,"available":94399746048,"internally_reserved":0,"allocated":783028224,"data_stored":782173482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":812251590},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1,6,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019219,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866128,"kb_used_data":245192,"kb_used_omap":1,"kb_used_meta":620926,"kb_avail":92879024,"statfs":{"total":95995035648,"available":95108120576,"internally_reserved":0,"allocated":251076608,"data_stored":250221730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":635828678},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650431,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":922948,"kb_used_data":698364,"kb_used_omap":4,"kb_used_meta":224379,"kb_avail":92822204,"statfs":{"total":95995035648,"available":95049936896,"internally_reserved":0,"allocated":715124736,"data_stored":714277924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":229764655},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879366,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376460,"kb_used_data":256772,"kb_used_omap":6,"kb_used_meta":119673,"kb_avail":93368692,"statfs":{"total":95995035648,"available":95609540608,"internally_reserved":0,"allocated":262934528,"data_stored":262080140,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":122545721},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435382,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263360,"kb_used_data":878520,"kb_used_omap":5,"kb_used_meta":384762,"kb_avail":92481792,"statfs":{"total":95995035648,"available":94701355008,"internally_reserved":0,"allocated":899604480,"data_stored":898760661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":393996641},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925957,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649092,"kb_used_data":364860,"kb_used_omap":5,"kb_used_meta":284218,"kb_avail":93096060,"statfs":{"total":95995035648,"available":95330365440,"internally_reserved":0,"allocated":373616640,"data_stored":372769798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":291039402},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":369688576,"data_stored":369688576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":870387712,"data_stored":870383640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":258146304,"data_stored":258146304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":683802624,"data_stored":683802624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":243990528,"data_stored":243990528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":758255616,"data_stored":758251544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":256049152,"data_stored":256049152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":737873920,"data_stored":737869848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:45:43.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:45:43.955 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:44.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:44.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:44.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:44.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:44.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:44.242 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:44 smithi042 ceph-mon[113926]: pgmap v612: 99 pgs: 99 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:45:44.242 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:44 smithi042 ceph-mon[113926]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:44.242 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:44 smithi042 ceph-mon[108570]: pgmap v612: 99 pgs: 99 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:45:44.242 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:44 smithi042 ceph-mon[108570]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:44.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:44.347 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:44 smithi159 ceph-mon[102519]: pgmap v612: 99 pgs: 99 active+clean; 1.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-13T14:45:44.347 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:44 smithi159 ceph-mon[102519]: from='client.34941 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:45:44.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:44.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:44.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:44.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:44.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:44.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:44.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:44.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:44.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:44.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:44.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:44.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:44.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:45.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:45.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:45.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:45.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:45.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:45.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:45.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:45.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:45.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:45.610 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:45.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:45.763 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:45.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:45.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:45.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:45.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:45.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:45.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:46.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:46.096 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:46.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:46 smithi042 ceph-mon[113926]: pgmap v613: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:45:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:46 smithi042 ceph-mon[108570]: pgmap v613: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:45:46.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:46.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:46.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:46.297 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:46 smithi159 ceph-mon[102519]: pgmap v613: 99 pgs: 99 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-13T14:45:46.350 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:46.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:46.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:46.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:46.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:46.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:46.689 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:46.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:46.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:46.923 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:46.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:46.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:47.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:47.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:47.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:47.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:47.332 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:47.375 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:47.477 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:47.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:47.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:47.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:47.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:47.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:47.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:47.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:47.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:47.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:47.926 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:47.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:48.028 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:48.049 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:48.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:48.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:48.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:47 smithi159 ceph-mon[102519]: pgmap v614: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:45:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:47 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:48.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:48.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[113926]: pgmap v614: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[108570]: pgmap v614: 99 pgs: 99 active+clean; 1.1 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 998 op/s 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:47 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:48.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:48.471 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:48.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:48.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:48.614 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:48.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:48.747 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:48.848 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:48.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:48.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:48.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.024 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:49.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:49.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:49.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:49.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:49.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:49.416 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:49.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:49.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:49.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:49.607 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:49.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:49.736 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:49.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:49.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:49.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:49.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:49.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:50.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:50.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:50.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[108570]: pgmap v615: 99 pgs: 99 active+clean; 1017 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:45:50.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3209186663' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.100 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:49] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-09-13T14:45:50.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[113926]: pgmap v615: 99 pgs: 99 active+clean; 1017 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:45:50.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3209186663' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:49 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.101 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:50.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:50.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.206 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:45:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:49 smithi159 ceph-mon[102519]: pgmap v615: 99 pgs: 99 active+clean; 1017 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-13T14:45:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3209186663' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:49 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:45:50.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:50.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:50.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:50.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.486 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:50.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:50.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:50.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:50.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:50.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:50.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:50.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:50.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:50.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:50.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:51.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.051 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:51.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[113926]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:45:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[108570]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:45:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:51.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:51.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:51.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:50 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:45:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:50 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:45:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:50 smithi159 ceph-mon[102519]: osdmap e174: 8 total, 8 up, 8 in 2024-09-13T14:45:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:45:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:45:51.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:51.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:51.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:51.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:51.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:51.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:51.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:51.576 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:45:51.577 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:45:51.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:51.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:51.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:45:51.622 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:45:51.648 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:45:51.649 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:45:51.684 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:45:51.685 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:45:51.707 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:51.716 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:45:51.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:45:51.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:51.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:51.762 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:45:51.762 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:45:51.777 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:51.805 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:45:51.805 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:45:51.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:51.854 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:45:51.854 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:45:51.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:51.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:51.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:52.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:52.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:52.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:51 smithi042 ceph-mon[113926]: pgmap v617: 99 pgs: 99 active+clean; 968 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:45:52.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3272258603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:52.075 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:51 smithi042 ceph-mon[108570]: pgmap v617: 99 pgs: 99 active+clean; 968 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:45:52.075 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3272258603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:52.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:52.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:51 smithi159 ceph-mon[102519]: pgmap v617: 99 pgs: 99 active+clean; 968 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T14:45:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3272258603' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:45:52.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:52.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:52.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:52.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:52.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:52.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:52.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:52.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:52.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:52.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:52.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:52.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:52.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:52.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:45:52.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:52.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:52.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:52.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:53.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:53.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:53.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:53.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:53.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:53.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:53.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:53.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:53.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:53.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:53.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:53.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:53.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:53.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:53.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:53.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:53.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:54.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:53 smithi042 ceph-mon[113926]: pgmap v618: 99 pgs: 99 active+clean; 792 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-13T14:45:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:53 smithi042 ceph-mon[108570]: pgmap v618: 99 pgs: 99 active+clean; 792 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-13T14:45:54.117 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:54.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:54.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:53 smithi159 ceph-mon[102519]: pgmap v618: 99 pgs: 99 active+clean; 792 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-13T14:45:54.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:54.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:54.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:54.520 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:54.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:54.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:54.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:54.689 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:54.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:54.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:54.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:54.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:54.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:55.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.192 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:55.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:55.363 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:55.465 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:55.503 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:55.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:55.633 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:55.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:55.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:55.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:55.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:56.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.046 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:56.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:56.148 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.184 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:56.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:55 smithi159 ceph-mon[102519]: pgmap v619: 99 pgs: 99 active+clean; 691 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-13T14:45:56.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.323 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:56.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:55 smithi042 ceph-mon[113926]: pgmap v619: 99 pgs: 99 active+clean; 691 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-13T14:45:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:55 smithi042 ceph-mon[108570]: pgmap v619: 99 pgs: 99 active+clean; 691 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-13T14:45:56.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:56.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:56.569 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.609 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:56.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:56.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:56.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:56.752 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:45:56.752 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:45:56.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:56.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:56.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:56.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:57.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:57.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:57.177 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:57.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:57.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:57.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:57.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:57.452 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:57.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:57.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:57.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:57.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:45:57.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:57.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:57.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:57.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:57.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:57.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:57.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:45:57.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:45:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:58.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:45:58.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:58.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:58.137 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:57 smithi042 ceph-mon[108570]: pgmap v620: 99 pgs: 99 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:45:58.138 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:57 smithi042 ceph-mon[113926]: pgmap v620: 99 pgs: 99 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:45:58.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:58.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:57 smithi159 ceph-mon[102519]: pgmap v620: 99 pgs: 99 active+clean; 537 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-13T14:45:58.265 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:58.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:58.284 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:58.284 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:58.361 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:45:58.366 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:58.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:58.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:45:58.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:45:58.556 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:45:58.558 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:45:58.622 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:58.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:58.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.821 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:58.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:58.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:45:58.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:58.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:45:59.061 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:59.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:59.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:45:59.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:45:59.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:45:59.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:45:59.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:59.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:59.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:59.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:45:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:59.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:45:59.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:45:59.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:45:59.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:45:59.883 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:45:59.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:45:59.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:45:59.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:45:59.984 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:45:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:45:59] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-09-13T14:46:00.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:00.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:00.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:00.154 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:00.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:00.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:00.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:59 smithi159 ceph-mon[102519]: pgmap v621: 99 pgs: 99 active+clean; 468 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-13T14:46:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:45:59 smithi159 ceph-mon[102519]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:00.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:00.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:00.328 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:59 smithi042 ceph-mon[108570]: pgmap v621: 99 pgs: 99 active+clean; 468 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-13T14:46:00.328 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:45:59 smithi042 ceph-mon[108570]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:00.328 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:59 smithi042 ceph-mon[113926]: pgmap v621: 99 pgs: 99 active+clean; 468 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-13T14:46:00.328 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:45:59 smithi042 ceph-mon[113926]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:46:00.367 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":621,"stamp":"2024-09-13T14:45:58.266646+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":491067389,"num_objects":7511,"num_object_clones":0,"num_object_copies":22533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7511,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":252473,"num_write_kb":8316578,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125966,"ondisk_log_size":125966,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5130424,"kb_used_data":1496504,"kb_used_omap":28,"kb_used_meta":3633635,"kb_avail":744830792,"statfs":{"total":767960285184,"available":762706731008,"internally_reserved":0,"allocated":1532420096,"data_stored":1525569850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":3720842641},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,3,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-575602688,"num_objects":-8783,"num_object_clones":0,"num_object_copies":-26349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8783,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006251"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.967888+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:45:49.967888+0000","last_peered":"2024-09-13T14:45:49.967888+0000","last_clean":"2024-09-13T14:45:49.967888+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:45:49.967888+0000","last_undegraded":"2024-09-13T14:45:49.967888+0000","last_fullsized":"2024-09-13T14:45:49.967888+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":144,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.962169+0000","last_change":"2024-09-13T14:43:35.907052+0000","last_active":"2024-09-13T14:45:49.962169+0000","last_peered":"2024-09-13T14:45:49.962169+0000","last_clean":"2024-09-13T14:45:49.962169+0000","last_became_active":"2024-09-13T14:43:35.906588+0000","last_became_peered":"2024-09-13T14:43:35.906588+0000","last_unstale":"2024-09-13T14:45:49.962169+0000","last_undegraded":"2024-09-13T14:45:49.962169+0000","last_fullsized":"2024-09-13T14:45:49.962169+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:56.521107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":138,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.018322+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:45:29.018322+0000","last_peered":"2024-09-13T14:45:29.018322+0000","last_clean":"2024-09-13T14:45:29.018322+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:45:29.018322+0000","last_undegraded":"2024-09-13T14:45:29.018322+0000","last_fullsized":"2024-09-13T14:45:29.018322+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":196,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.974958+0000","last_change":"2024-09-13T14:40:37.187655+0000","last_active":"2024-09-13T14:45:49.974958+0000","last_peered":"2024-09-13T14:45:49.974958+0000","last_clean":"2024-09-13T14:45:49.974958+0000","last_became_active":"2024-09-13T14:40:37.187315+0000","last_became_peered":"2024-09-13T14:40:37.187315+0000","last_unstale":"2024-09-13T14:45:49.974958+0000","last_undegraded":"2024-09-13T14:45:49.974958+0000","last_fullsized":"2024-09-13T14:45:49.974958+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:31:19.898423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":197,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975366+0000","last_change":"2024-09-13T14:40:37.187575+0000","last_active":"2024-09-13T14:45:49.975366+0000","last_peered":"2024-09-13T14:45:49.975366+0000","last_clean":"2024-09-13T14:45:49.975366+0000","last_became_active":"2024-09-13T14:40:37.187178+0000","last_became_peered":"2024-09-13T14:40:37.187178+0000","last_unstale":"2024-09-13T14:45:49.975366+0000","last_undegraded":"2024-09-13T14:45:49.975366+0000","last_fullsized":"2024-09-13T14:45:49.975366+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:47.757515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":30,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.958745+0000","last_change":"2024-09-13T14:43:35.907589+0000","last_active":"2024-09-13T14:45:49.958745+0000","last_peered":"2024-09-13T14:45:49.958745+0000","last_clean":"2024-09-13T14:45:49.958745+0000","last_became_active":"2024-09-13T14:43:35.907254+0000","last_became_peered":"2024-09-13T14:43:35.907254+0000","last_unstale":"2024-09-13T14:45:49.958745+0000","last_undegraded":"2024-09-13T14:45:49.958745+0000","last_fullsized":"2024-09-13T14:45:49.958745+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:13.369649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":194,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959038+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:45:49.959038+0000","last_peered":"2024-09-13T14:45:49.959038+0000","last_clean":"2024-09-13T14:45:49.959038+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:45:49.959038+0000","last_undegraded":"2024-09-13T14:45:49.959038+0000","last_fullsized":"2024-09-13T14:45:49.959038+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":148,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021119+0000","last_change":"2024-09-13T14:34:44.816114+0000","last_active":"2024-09-13T14:45:29.021119+0000","last_peered":"2024-09-13T14:45:29.021119+0000","last_clean":"2024-09-13T14:45:29.021119+0000","last_became_active":"2024-09-13T14:34:44.815762+0000","last_became_peered":"2024-09-13T14:34:44.815762+0000","last_unstale":"2024-09-13T14:45:29.021119+0000","last_undegraded":"2024-09-13T14:45:29.021119+0000","last_fullsized":"2024-09-13T14:45:29.021119+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:32.160006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"174'15608","reported_seq":15691,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:53.702277+0000","last_change":"2024-09-13T14:42:07.783733+0000","last_active":"2024-09-13T14:45:53.702277+0000","last_peered":"2024-09-13T14:45:53.702277+0000","last_clean":"2024-09-13T14:45:53.702277+0000","last_became_active":"2024-09-13T14:42:07.783466+0000","last_became_peered":"2024-09-13T14:42:07.783466+0000","last_unstale":"2024-09-13T14:45:53.702277+0000","last_undegraded":"2024-09-13T14:45:53.702277+0000","last_fullsized":"2024-09-13T14:45:53.702277+0000","mapping_epoch":139,"log_start":"153'6500","ondisk_log_start":"153'6500","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:20.205570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39845888,"num_objects":608,"num_object_clones":0,"num_object_copies":1824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15608,"num_write_kb":518912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":42,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959958+0000","last_change":"2024-09-13T14:43:35.907886+0000","last_active":"2024-09-13T14:45:49.959958+0000","last_peered":"2024-09-13T14:45:49.959958+0000","last_clean":"2024-09-13T14:45:49.959958+0000","last_became_active":"2024-09-13T14:43:35.906651+0000","last_became_peered":"2024-09-13T14:43:35.906651+0000","last_unstale":"2024-09-13T14:45:49.959958+0000","last_undegraded":"2024-09-13T14:45:49.959958+0000","last_fullsized":"2024-09-13T14:45:49.959958+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:48:36.982080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":124,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017651+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:45:29.017651+0000","last_peered":"2024-09-13T14:45:29.017651+0000","last_clean":"2024-09-13T14:45:29.017651+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:45:29.017651+0000","last_undegraded":"2024-09-13T14:45:29.017651+0000","last_fullsized":"2024-09-13T14:45:29.017651+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"174'15708","reported_seq":15791,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.054484+0000","last_change":"2024-09-13T14:42:07.780479+0000","last_active":"2024-09-13T14:45:58.054484+0000","last_peered":"2024-09-13T14:45:58.054484+0000","last_clean":"2024-09-13T14:45:58.054484+0000","last_became_active":"2024-09-13T14:42:07.780123+0000","last_became_peered":"2024-09-13T14:42:07.780123+0000","last_unstale":"2024-09-13T14:45:58.054484+0000","last_undegraded":"2024-09-13T14:45:58.054484+0000","last_fullsized":"2024-09-13T14:45:58.054484+0000","mapping_epoch":139,"log_start":"158'8700","ondisk_log_start":"158'8700","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:12:11.339751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21233664,"num_objects":324,"num_object_clones":0,"num_object_copies":972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15708,"num_write_kb":513024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":42,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960029+0000","last_change":"2024-09-13T14:43:35.905294+0000","last_active":"2024-09-13T14:45:49.960029+0000","last_peered":"2024-09-13T14:45:49.960029+0000","last_clean":"2024-09-13T14:45:49.960029+0000","last_became_active":"2024-09-13T14:43:35.904851+0000","last_became_peered":"2024-09-13T14:43:35.904851+0000","last_unstale":"2024-09-13T14:45:49.960029+0000","last_undegraded":"2024-09-13T14:45:49.960029+0000","last_fullsized":"2024-09-13T14:45:49.960029+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:21.591225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019245+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:45:29.019245+0000","last_peered":"2024-09-13T14:45:29.019245+0000","last_clean":"2024-09-13T14:45:29.019245+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:45:29.019245+0000","last_undegraded":"2024-09-13T14:45:29.019245+0000","last_fullsized":"2024-09-13T14:45:29.019245+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"174'15747","reported_seq":15792,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:56.134912+0000","last_change":"2024-09-13T14:42:07.781173+0000","last_active":"2024-09-13T14:45:56.134912+0000","last_peered":"2024-09-13T14:45:56.134912+0000","last_clean":"2024-09-13T14:45:56.134912+0000","last_became_active":"2024-09-13T14:42:07.780794+0000","last_became_peered":"2024-09-13T14:42:07.780794+0000","last_unstale":"2024-09-13T14:45:56.134912+0000","last_undegraded":"2024-09-13T14:45:56.134912+0000","last_fullsized":"2024-09-13T14:45:56.134912+0000","mapping_epoch":139,"log_start":"155'6900","ondisk_log_start":"155'6900","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8847,"log_dups_size":0,"ondisk_log_size":8847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:00:03.397389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15747,"num_write_kb":518848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":42,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.962599+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:45:49.962599+0000","last_peered":"2024-09-13T14:45:49.962599+0000","last_clean":"2024-09-13T14:45:49.962599+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:45:49.962599+0000","last_undegraded":"2024-09-13T14:45:49.962599+0000","last_fullsized":"2024-09-13T14:45:49.962599+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":140,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021132+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:45:29.021132+0000","last_peered":"2024-09-13T14:45:29.021132+0000","last_clean":"2024-09-13T14:45:29.021132+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:45:29.021132+0000","last_undegraded":"2024-09-13T14:45:29.021132+0000","last_fullsized":"2024-09-13T14:45:29.021132+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"174'15465","reported_seq":15510,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:56.141740+0000","last_change":"2024-09-13T14:42:07.781817+0000","last_active":"2024-09-13T14:45:56.141740+0000","last_peered":"2024-09-13T14:45:56.141740+0000","last_clean":"2024-09-13T14:45:56.141740+0000","last_became_active":"2024-09-13T14:42:07.781534+0000","last_became_peered":"2024-09-13T14:42:07.781534+0000","last_unstale":"2024-09-13T14:45:56.141740+0000","last_undegraded":"2024-09-13T14:45:56.141740+0000","last_fullsized":"2024-09-13T14:45:56.141740+0000","mapping_epoch":139,"log_start":"155'6600","ondisk_log_start":"155'6600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8865,"log_dups_size":0,"ondisk_log_size":8865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:22:27.372521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28377088,"num_objects":433,"num_object_clones":0,"num_object_copies":1299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15465,"num_write_kb":508736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:31.057585+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:45:31.057585+0000","last_peered":"2024-09-13T14:45:31.057585+0000","last_clean":"2024-09-13T14:45:31.057585+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:45:31.057585+0000","last_undegraded":"2024-09-13T14:45:31.057585+0000","last_fullsized":"2024-09-13T14:45:31.057585+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":216,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.981198+0000","last_change":"2024-09-13T14:41:21.825172+0000","last_active":"2024-09-13T14:45:49.981198+0000","last_peered":"2024-09-13T14:45:49.981198+0000","last_clean":"2024-09-13T14:45:49.981198+0000","last_became_active":"2024-09-13T14:41:21.824794+0000","last_became_peered":"2024-09-13T14:41:21.824794+0000","last_unstale":"2024-09-13T14:45:49.981198+0000","last_undegraded":"2024-09-13T14:45:49.981198+0000","last_fullsized":"2024-09-13T14:45:49.981198+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:29:21.127447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":518,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.957419+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:45:49.957419+0000","last_peered":"2024-09-13T14:45:49.957419+0000","last_clean":"2024-09-13T14:45:49.957419+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:45:49.957419+0000","last_undegraded":"2024-09-13T14:45:49.957419+0000","last_fullsized":"2024-09-13T14:45:49.957419+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.1","version":"174'15811","reported_seq":15894,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:55.761994+0000","last_change":"2024-09-13T14:42:07.942573+0000","last_active":"2024-09-13T14:45:55.761994+0000","last_peered":"2024-09-13T14:45:55.761994+0000","last_clean":"2024-09-13T14:45:55.761994+0000","last_became_active":"2024-09-13T14:42:07.942229+0000","last_became_peered":"2024-09-13T14:42:07.942229+0000","last_unstale":"2024-09-13T14:45:55.761994+0000","last_undegraded":"2024-09-13T14:45:55.761994+0000","last_fullsized":"2024-09-13T14:45:55.761994+0000","mapping_epoch":139,"log_start":"159'9500","ondisk_log_start":"159'9500","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:43:10.238039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29949952,"num_objects":457,"num_object_clones":0,"num_object_copies":1371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15811,"num_write_kb":520576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":156,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.971144+0000","last_change":"2024-09-13T14:40:37.184229+0000","last_active":"2024-09-13T14:45:49.971144+0000","last_peered":"2024-09-13T14:45:49.971144+0000","last_clean":"2024-09-13T14:45:49.971144+0000","last_became_active":"2024-09-13T14:40:37.183563+0000","last_became_peered":"2024-09-13T14:40:37.183563+0000","last_unstale":"2024-09-13T14:45:49.971144+0000","last_undegraded":"2024-09-13T14:45:49.971144+0000","last_fullsized":"2024-09-13T14:45:49.971144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:28:37.620656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:31.336497+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:45:31.336497+0000","last_peered":"2024-09-13T14:45:31.336497+0000","last_clean":"2024-09-13T14:45:31.336497+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:45:31.336497+0000","last_undegraded":"2024-09-13T14:45:31.336497+0000","last_fullsized":"2024-09-13T14:45:31.336497+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":409,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017270+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:45:29.017270+0000","last_peered":"2024-09-13T14:45:29.017270+0000","last_clean":"2024-09-13T14:45:29.017270+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:45:29.017270+0000","last_undegraded":"2024-09-13T14:45:29.017270+0000","last_fullsized":"2024-09-13T14:45:29.017270+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.0","version":"174'15853","reported_seq":15936,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.108623+0000","last_change":"2024-09-13T14:42:07.942557+0000","last_active":"2024-09-13T14:45:58.108623+0000","last_peered":"2024-09-13T14:45:58.108623+0000","last_clean":"2024-09-13T14:45:58.108623+0000","last_became_active":"2024-09-13T14:42:07.942184+0000","last_became_peered":"2024-09-13T14:42:07.942184+0000","last_unstale":"2024-09-13T14:45:58.108623+0000","last_undegraded":"2024-09-13T14:45:58.108623+0000","last_fullsized":"2024-09-13T14:45:58.108623+0000","mapping_epoch":139,"log_start":"158'8800","ondisk_log_start":"158'8800","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":7053,"log_dups_size":0,"ondisk_log_size":7053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:39:04.365348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23265280,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15853,"num_write_kb":518656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021251+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:45:29.021251+0000","last_peered":"2024-09-13T14:45:29.021251+0000","last_clean":"2024-09-13T14:45:29.021251+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:45:29.021251+0000","last_undegraded":"2024-09-13T14:45:29.021251+0000","last_fullsized":"2024-09-13T14:45:29.021251+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:31.336547+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:45:31.336547+0000","last_peered":"2024-09-13T14:45:31.336547+0000","last_clean":"2024-09-13T14:45:31.336547+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:45:31.336547+0000","last_undegraded":"2024-09-13T14:45:31.336547+0000","last_fullsized":"2024-09-13T14:45:31.336547+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"174'16160","reported_seq":16230,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:55.717514+0000","last_change":"2024-09-13T14:43:35.927511+0000","last_active":"2024-09-13T14:45:55.717514+0000","last_peered":"2024-09-13T14:45:55.717514+0000","last_clean":"2024-09-13T14:45:55.717514+0000","last_became_active":"2024-09-13T14:43:35.907655+0000","last_became_peered":"2024-09-13T14:43:35.907655+0000","last_unstale":"2024-09-13T14:45:55.717514+0000","last_undegraded":"2024-09-13T14:45:55.717514+0000","last_fullsized":"2024-09-13T14:45:55.717514+0000","mapping_epoch":156,"log_start":"160'9844","ondisk_log_start":"160'9844","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:43.071844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30015488,"num_objects":458,"num_object_clones":0,"num_object_copies":1374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":16160,"num_write_kb":531776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019317+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:45:29.019317+0000","last_peered":"2024-09-13T14:45:29.019317+0000","last_clean":"2024-09-13T14:45:29.019317+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:45:29.019317+0000","last_undegraded":"2024-09-13T14:45:29.019317+0000","last_fullsized":"2024-09-13T14:45:29.019317+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":149,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:36.343053+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:45:36.343053+0000","last_peered":"2024-09-13T14:45:36.343053+0000","last_clean":"2024-09-13T14:45:36.343053+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:45:36.343053+0000","last_undegraded":"2024-09-13T14:45:36.343053+0000","last_fullsized":"2024-09-13T14:45:36.343053+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":409,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017325+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:45:29.017325+0000","last_peered":"2024-09-13T14:45:29.017325+0000","last_clean":"2024-09-13T14:45:29.017325+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:45:29.017325+0000","last_undegraded":"2024-09-13T14:45:29.017325+0000","last_fullsized":"2024-09-13T14:45:29.017325+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.2","version":"174'15657","reported_seq":15740,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:55.754328+0000","last_change":"2024-09-13T14:42:07.780605+0000","last_active":"2024-09-13T14:45:55.754328+0000","last_peered":"2024-09-13T14:45:55.754328+0000","last_clean":"2024-09-13T14:45:55.754328+0000","last_became_active":"2024-09-13T14:42:07.780309+0000","last_became_peered":"2024-09-13T14:42:07.780309+0000","last_unstale":"2024-09-13T14:45:55.754328+0000","last_undegraded":"2024-09-13T14:45:55.754328+0000","last_fullsized":"2024-09-13T14:45:55.754328+0000","mapping_epoch":139,"log_start":"159'9400","ondisk_log_start":"159'9400","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6257,"log_dups_size":0,"ondisk_log_size":6257,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:29:17.130563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32440320,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15657,"num_write_kb":516864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":170,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.969330+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:45:49.969330+0000","last_peered":"2024-09-13T14:45:49.969330+0000","last_clean":"2024-09-13T14:45:49.969330+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:45:49.969330+0000","last_undegraded":"2024-09-13T14:45:49.969330+0000","last_fullsized":"2024-09-13T14:45:49.969330+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.963430+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:45:49.963430+0000","last_peered":"2024-09-13T14:45:49.963430+0000","last_clean":"2024-09-13T14:45:49.963430+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:45:49.963430+0000","last_undegraded":"2024-09-13T14:45:49.963430+0000","last_fullsized":"2024-09-13T14:45:49.963430+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"174'15873","reported_seq":15954,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:55.773749+0000","last_change":"2024-09-13T14:42:07.782112+0000","last_active":"2024-09-13T14:45:55.773749+0000","last_peered":"2024-09-13T14:45:55.773749+0000","last_clean":"2024-09-13T14:45:55.773749+0000","last_became_active":"2024-09-13T14:42:07.781780+0000","last_became_peered":"2024-09-13T14:42:07.781780+0000","last_unstale":"2024-09-13T14:45:55.773749+0000","last_undegraded":"2024-09-13T14:45:55.773749+0000","last_fullsized":"2024-09-13T14:45:55.773749+0000","mapping_epoch":139,"log_start":"160'9600","ondisk_log_start":"160'9600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:47:55.236172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15873,"num_write_kb":523840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":415,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.987070+0000","last_change":"2024-09-13T14:43:37.621179+0000","last_active":"2024-09-13T14:45:49.987070+0000","last_peered":"2024-09-13T14:45:49.987070+0000","last_clean":"2024-09-13T14:45:49.987070+0000","last_became_active":"2024-09-13T14:43:35.909308+0000","last_became_peered":"2024-09-13T14:43:35.909308+0000","last_unstale":"2024-09-13T14:45:49.987070+0000","last_undegraded":"2024-09-13T14:45:49.987070+0000","last_fullsized":"2024-09-13T14:45:49.987070+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:37.621104+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:37.621104+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:07:19.210888+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.018058+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:45:29.018058+0000","last_peered":"2024-09-13T14:45:29.018058+0000","last_clean":"2024-09-13T14:45:29.018058+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:45:29.018058+0000","last_undegraded":"2024-09-13T14:45:29.018058+0000","last_fullsized":"2024-09-13T14:45:29.018058+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.957914+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:45:49.957914+0000","last_peered":"2024-09-13T14:45:49.957914+0000","last_clean":"2024-09-13T14:45:49.957914+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:45:49.957914+0000","last_undegraded":"2024-09-13T14:45:49.957914+0000","last_fullsized":"2024-09-13T14:45:49.957914+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":126,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.966690+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:45:49.966690+0000","last_peered":"2024-09-13T14:45:49.966690+0000","last_clean":"2024-09-13T14:45:49.966690+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:45:49.966690+0000","last_undegraded":"2024-09-13T14:45:49.966690+0000","last_fullsized":"2024-09-13T14:45:49.966690+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":428,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.961971+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:45:49.961971+0000","last_peered":"2024-09-13T14:45:49.961971+0000","last_clean":"2024-09-13T14:45:49.961971+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:45:49.961971+0000","last_undegraded":"2024-09-13T14:45:49.961971+0000","last_fullsized":"2024-09-13T14:45:49.961971+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.b","version":"174'15745","reported_seq":15828,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:53.740086+0000","last_change":"2024-09-13T14:42:07.942512+0000","last_active":"2024-09-13T14:45:53.740086+0000","last_peered":"2024-09-13T14:45:53.740086+0000","last_clean":"2024-09-13T14:45:53.740086+0000","last_became_active":"2024-09-13T14:42:07.942191+0000","last_became_peered":"2024-09-13T14:42:07.942191+0000","last_unstale":"2024-09-13T14:45:53.740086+0000","last_undegraded":"2024-09-13T14:45:53.740086+0000","last_fullsized":"2024-09-13T14:45:53.740086+0000","mapping_epoch":139,"log_start":"153'6600","ondisk_log_start":"153'6600","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9145,"log_dups_size":0,"ondisk_log_size":9145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:56.446026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38731776,"num_objects":591,"num_object_clones":0,"num_object_copies":1773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15745,"num_write_kb":522752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":141,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959750+0000","last_change":"2024-09-13T14:43:35.907156+0000","last_active":"2024-09-13T14:45:49.959750+0000","last_peered":"2024-09-13T14:45:49.959750+0000","last_clean":"2024-09-13T14:45:49.959750+0000","last_became_active":"2024-09-13T14:43:35.906893+0000","last_became_peered":"2024-09-13T14:43:35.906893+0000","last_unstale":"2024-09-13T14:45:49.959750+0000","last_undegraded":"2024-09-13T14:45:49.959750+0000","last_fullsized":"2024-09-13T14:45:49.959750+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:36.233310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":196,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.974785+0000","last_change":"2024-09-13T14:40:37.187388+0000","last_active":"2024-09-13T14:45:49.974785+0000","last_peered":"2024-09-13T14:45:49.974785+0000","last_clean":"2024-09-13T14:45:49.974785+0000","last_became_active":"2024-09-13T14:40:37.187025+0000","last_became_peered":"2024-09-13T14:40:37.187025+0000","last_unstale":"2024-09-13T14:45:49.974785+0000","last_undegraded":"2024-09-13T14:45:49.974785+0000","last_fullsized":"2024-09-13T14:45:49.974785+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:25:02.883609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":209,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975200+0000","last_change":"2024-09-13T14:40:02.372242+0000","last_active":"2024-09-13T14:45:49.975200+0000","last_peered":"2024-09-13T14:45:49.975200+0000","last_clean":"2024-09-13T14:45:49.975200+0000","last_became_active":"2024-09-13T14:40:02.371903+0000","last_became_peered":"2024-09-13T14:40:02.371903+0000","last_unstale":"2024-09-13T14:45:49.975200+0000","last_undegraded":"2024-09-13T14:45:49.975200+0000","last_fullsized":"2024-09-13T14:45:49.975200+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:13.999618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":415,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.974786+0000","last_change":"2024-09-13T14:43:36.606414+0000","last_active":"2024-09-13T14:45:49.974786+0000","last_peered":"2024-09-13T14:45:49.974786+0000","last_clean":"2024-09-13T14:45:49.974786+0000","last_became_active":"2024-09-13T14:43:35.907350+0000","last_became_peered":"2024-09-13T14:43:35.907350+0000","last_unstale":"2024-09-13T14:45:49.974786+0000","last_undegraded":"2024-09-13T14:45:49.974786+0000","last_fullsized":"2024-09-13T14:45:49.974786+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:36.606328+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:36.606328+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:16:34.516165+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.a","version":"174'15866","reported_seq":15911,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:56.158078+0000","last_change":"2024-09-13T14:42:07.781099+0000","last_active":"2024-09-13T14:45:56.158078+0000","last_peered":"2024-09-13T14:45:56.158078+0000","last_clean":"2024-09-13T14:45:56.158078+0000","last_became_active":"2024-09-13T14:42:07.780786+0000","last_became_peered":"2024-09-13T14:42:07.780786+0000","last_unstale":"2024-09-13T14:45:56.158078+0000","last_undegraded":"2024-09-13T14:45:56.158078+0000","last_fullsized":"2024-09-13T14:45:56.158078+0000","mapping_epoch":139,"log_start":"155'7000","ondisk_log_start":"155'7000","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8866,"log_dups_size":0,"ondisk_log_size":8866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:24.082388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15866,"num_write_kb":523200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":192,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.954408+0000","last_change":"2024-09-13T14:40:37.180085+0000","last_active":"2024-09-13T14:45:49.954408+0000","last_peered":"2024-09-13T14:45:49.954408+0000","last_clean":"2024-09-13T14:45:49.954408+0000","last_became_active":"2024-09-13T14:40:37.179657+0000","last_became_peered":"2024-09-13T14:40:37.179657+0000","last_unstale":"2024-09-13T14:45:49.954408+0000","last_undegraded":"2024-09-13T14:45:49.954408+0000","last_fullsized":"2024-09-13T14:45:49.954408+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:10:11.654508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":167,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959990+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:45:49.959990+0000","last_peered":"2024-09-13T14:45:49.959990+0000","last_clean":"2024-09-13T14:45:49.959990+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:45:49.959990+0000","last_undegraded":"2024-09-13T14:45:49.959990+0000","last_fullsized":"2024-09-13T14:45:49.959990+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":203,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.963293+0000","last_change":"2024-09-13T14:35:58.854407+0000","last_active":"2024-09-13T14:45:49.963293+0000","last_peered":"2024-09-13T14:45:49.963293+0000","last_clean":"2024-09-13T14:45:49.963293+0000","last_became_active":"2024-09-13T14:35:58.854049+0000","last_became_peered":"2024-09-13T14:35:58.854049+0000","last_unstale":"2024-09-13T14:45:49.963293+0000","last_undegraded":"2024-09-13T14:45:49.963293+0000","last_fullsized":"2024-09-13T14:45:49.963293+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:44.475066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":409,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017818+0000","last_change":"2024-09-13T14:42:56.295979+0000","last_active":"2024-09-13T14:45:29.017818+0000","last_peered":"2024-09-13T14:45:29.017818+0000","last_clean":"2024-09-13T14:45:29.017818+0000","last_became_active":"2024-09-13T14:42:56.290769+0000","last_became_peered":"2024-09-13T14:42:56.290769+0000","last_unstale":"2024-09-13T14:45:29.017818+0000","last_undegraded":"2024-09-13T14:45:29.017818+0000","last_fullsized":"2024-09-13T14:45:29.017818+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:27.386430+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"174'15511","reported_seq":15594,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:53.717131+0000","last_change":"2024-09-13T14:42:07.783147+0000","last_active":"2024-09-13T14:45:53.717131+0000","last_peered":"2024-09-13T14:45:53.717131+0000","last_clean":"2024-09-13T14:45:53.717131+0000","last_became_active":"2024-09-13T14:42:07.782831+0000","last_became_peered":"2024-09-13T14:42:07.782831+0000","last_unstale":"2024-09-13T14:45:53.717131+0000","last_undegraded":"2024-09-13T14:45:53.717131+0000","last_fullsized":"2024-09-13T14:45:53.717131+0000","mapping_epoch":139,"log_start":"153'6400","ondisk_log_start":"153'6400","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":9111,"log_dups_size":0,"ondisk_log_size":9111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:08:23.396196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37814272,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15511,"num_write_kb":514816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":155,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.017783+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:45:29.017783+0000","last_peered":"2024-09-13T14:45:29.017783+0000","last_clean":"2024-09-13T14:45:29.017783+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:45:29.017783+0000","last_undegraded":"2024-09-13T14:45:29.017783+0000","last_fullsized":"2024-09-13T14:45:29.017783+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":92,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.971010+0000","last_change":"2024-09-13T14:40:38.175512+0000","last_active":"2024-09-13T14:45:49.971010+0000","last_peered":"2024-09-13T14:45:49.971010+0000","last_clean":"2024-09-13T14:45:49.971010+0000","last_became_active":"2024-09-13T14:40:38.174244+0000","last_became_peered":"2024-09-13T14:40:38.174244+0000","last_unstale":"2024-09-13T14:45:49.971010+0000","last_undegraded":"2024-09-13T14:45:49.971010+0000","last_fullsized":"2024-09-13T14:45:49.971010+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:58.244469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":179,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960641+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:45:49.960641+0000","last_peered":"2024-09-13T14:45:49.960641+0000","last_clean":"2024-09-13T14:45:49.960641+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:45:49.960641+0000","last_undegraded":"2024-09-13T14:45:49.960641+0000","last_fullsized":"2024-09-13T14:45:49.960641+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":417,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021475+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:45:29.021475+0000","last_peered":"2024-09-13T14:45:29.021475+0000","last_clean":"2024-09-13T14:45:29.021475+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:45:29.021475+0000","last_undegraded":"2024-09-13T14:45:29.021475+0000","last_fullsized":"2024-09-13T14:45:29.021475+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.f","version":"174'15820","reported_seq":15865,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:56.170048+0000","last_change":"2024-09-13T14:42:07.781837+0000","last_active":"2024-09-13T14:45:56.170048+0000","last_peered":"2024-09-13T14:45:56.170048+0000","last_clean":"2024-09-13T14:45:56.170048+0000","last_became_active":"2024-09-13T14:42:07.781584+0000","last_became_peered":"2024-09-13T14:42:07.781584+0000","last_unstale":"2024-09-13T14:45:56.170048+0000","last_undegraded":"2024-09-13T14:45:56.170048+0000","last_fullsized":"2024-09-13T14:45:56.170048+0000","mapping_epoch":139,"log_start":"155'6900","ondisk_log_start":"155'6900","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8920,"log_dups_size":0,"ondisk_log_size":8920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:54:21.323962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33554432,"num_objects":512,"num_object_clones":0,"num_object_copies":1536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15820,"num_write_kb":522624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":156,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.963519+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:45:49.963519+0000","last_peered":"2024-09-13T14:45:49.963519+0000","last_clean":"2024-09-13T14:45:49.963519+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:45:49.963519+0000","last_undegraded":"2024-09-13T14:45:49.963519+0000","last_fullsized":"2024-09-13T14:45:49.963519+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":217,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960823+0000","last_change":"2024-09-13T14:35:58.846330+0000","last_active":"2024-09-13T14:45:49.960823+0000","last_peered":"2024-09-13T14:45:49.960823+0000","last_clean":"2024-09-13T14:45:49.960823+0000","last_became_active":"2024-09-13T14:35:58.845985+0000","last_became_peered":"2024-09-13T14:35:58.845985+0000","last_unstale":"2024-09-13T14:45:49.960823+0000","last_undegraded":"2024-09-13T14:45:49.960823+0000","last_fullsized":"2024-09-13T14:45:49.960823+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:11.453182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":88,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960173+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:45:49.960173+0000","last_peered":"2024-09-13T14:45:49.960173+0000","last_clean":"2024-09-13T14:45:49.960173+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:45:49.960173+0000","last_undegraded":"2024-09-13T14:45:49.960173+0000","last_fullsized":"2024-09-13T14:45:49.960173+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":417,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.021456+0000","last_change":"2024-09-13T14:42:57.907376+0000","last_active":"2024-09-13T14:45:29.021456+0000","last_peered":"2024-09-13T14:45:29.021456+0000","last_clean":"2024-09-13T14:45:29.021456+0000","last_became_active":"2024-09-13T14:42:56.286461+0000","last_became_peered":"2024-09-13T14:42:56.286461+0000","last_unstale":"2024-09-13T14:45:29.021456+0000","last_undegraded":"2024-09-13T14:45:29.021456+0000","last_fullsized":"2024-09-13T14:45:29.021456+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:15:27.858330+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.e","version":"174'15880","reported_seq":15932,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:56.336431+0000","last_change":"2024-09-13T14:43:35.917618+0000","last_active":"2024-09-13T14:45:56.336431+0000","last_peered":"2024-09-13T14:45:56.336431+0000","last_clean":"2024-09-13T14:45:56.336431+0000","last_became_active":"2024-09-13T14:43:35.904748+0000","last_became_peered":"2024-09-13T14:43:35.904748+0000","last_unstale":"2024-09-13T14:45:56.336431+0000","last_undegraded":"2024-09-13T14:45:56.336431+0000","last_fullsized":"2024-09-13T14:45:56.336431+0000","mapping_epoch":156,"log_start":"155'7000","ondisk_log_start":"155'7000","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":8880,"log_dups_size":0,"ondisk_log_size":8880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:17.729503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15880,"num_write_kb":521856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":166,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960716+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:45:49.960716+0000","last_peered":"2024-09-13T14:45:49.960716+0000","last_clean":"2024-09-13T14:45:49.960716+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:45:49.960716+0000","last_undegraded":"2024-09-13T14:45:49.960716+0000","last_fullsized":"2024-09-13T14:45:49.960716+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":110,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.974947+0000","last_change":"2024-09-13T14:43:35.908853+0000","last_active":"2024-09-13T14:45:49.974947+0000","last_peered":"2024-09-13T14:45:49.974947+0000","last_clean":"2024-09-13T14:45:49.974947+0000","last_became_active":"2024-09-13T14:43:35.908544+0000","last_became_peered":"2024-09-13T14:43:35.908544+0000","last_unstale":"2024-09-13T14:45:49.974947+0000","last_undegraded":"2024-09-13T14:45:49.974947+0000","last_fullsized":"2024-09-13T14:45:49.974947+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:10:13.786745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":187,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.981346+0000","last_change":"2024-09-13T14:34:44.819831+0000","last_active":"2024-09-13T14:45:49.981346+0000","last_peered":"2024-09-13T14:45:49.981346+0000","last_clean":"2024-09-13T14:45:49.981346+0000","last_became_active":"2024-09-13T14:34:44.818945+0000","last_became_peered":"2024-09-13T14:34:44.818945+0000","last_unstale":"2024-09-13T14:45:49.981346+0000","last_undegraded":"2024-09-13T14:45:49.981346+0000","last_fullsized":"2024-09-13T14:45:49.981346+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:46.623919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":409,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019752+0000","last_change":"2024-09-13T14:43:00.868011+0000","last_active":"2024-09-13T14:45:29.019752+0000","last_peered":"2024-09-13T14:45:29.019752+0000","last_clean":"2024-09-13T14:45:29.019752+0000","last_became_active":"2024-09-13T14:42:56.283775+0000","last_became_peered":"2024-09-13T14:42:56.283775+0000","last_unstale":"2024-09-13T14:45:29.019752+0000","last_undegraded":"2024-09-13T14:45:29.019752+0000","last_fullsized":"2024-09-13T14:45:29.019752+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:00.867870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:00.867870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:25.806175+0000","scrub_duration":0.0043830680000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.d","version":"174'15876","reported_seq":15948,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:58.080663+0000","last_change":"2024-09-13T14:43:35.914172+0000","last_active":"2024-09-13T14:45:58.080663+0000","last_peered":"2024-09-13T14:45:58.080663+0000","last_clean":"2024-09-13T14:45:58.080663+0000","last_became_active":"2024-09-13T14:43:35.905309+0000","last_became_peered":"2024-09-13T14:43:35.905309+0000","last_unstale":"2024-09-13T14:45:58.080663+0000","last_undegraded":"2024-09-13T14:45:58.080663+0000","last_fullsized":"2024-09-13T14:45:58.080663+0000","mapping_epoch":156,"log_start":"158'8897","ondisk_log_start":"158'8897","created":139,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:18:30.311540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22478872,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15876,"num_write_kb":518977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":156,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959211+0000","last_change":"2024-09-13T14:40:37.186269+0000","last_active":"2024-09-13T14:45:49.959211+0000","last_peered":"2024-09-13T14:45:49.959211+0000","last_clean":"2024-09-13T14:45:49.959211+0000","last_became_active":"2024-09-13T14:40:37.185558+0000","last_became_peered":"2024-09-13T14:40:37.185558+0000","last_unstale":"2024-09-13T14:45:49.959211+0000","last_undegraded":"2024-09-13T14:45:49.959211+0000","last_fullsized":"2024-09-13T14:45:49.959211+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:06.713216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":167,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.964514+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:45:49.964514+0000","last_peered":"2024-09-13T14:45:49.964514+0000","last_clean":"2024-09-13T14:45:49.964514+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:45:49.964514+0000","last_undegraded":"2024-09-13T14:45:49.964514+0000","last_fullsized":"2024-09-13T14:45:49.964514+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":153,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.969070+0000","last_change":"2024-09-13T14:40:37.184432+0000","last_active":"2024-09-13T14:45:49.969070+0000","last_peered":"2024-09-13T14:45:49.969070+0000","last_clean":"2024-09-13T14:45:49.969070+0000","last_became_active":"2024-09-13T14:40:37.183736+0000","last_became_peered":"2024-09-13T14:40:37.183736+0000","last_unstale":"2024-09-13T14:45:49.969070+0000","last_undegraded":"2024-09-13T14:45:49.969070+0000","last_fullsized":"2024-09-13T14:45:49.969070+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:01.816421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":428,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.967563+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:45:49.967563+0000","last_peered":"2024-09-13T14:45:49.967563+0000","last_clean":"2024-09-13T14:45:49.967563+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:45:49.967563+0000","last_undegraded":"2024-09-13T14:45:49.967563+0000","last_fullsized":"2024-09-13T14:45:49.967563+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"174'15829","reported_seq":15912,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:55.757545+0000","last_change":"2024-09-13T14:42:07.782967+0000","last_active":"2024-09-13T14:45:55.757545+0000","last_peered":"2024-09-13T14:45:55.757545+0000","last_clean":"2024-09-13T14:45:55.757545+0000","last_became_active":"2024-09-13T14:42:07.782365+0000","last_became_peered":"2024-09-13T14:42:07.782365+0000","last_unstale":"2024-09-13T14:45:55.757545+0000","last_undegraded":"2024-09-13T14:45:55.757545+0000","last_fullsized":"2024-09-13T14:45:55.757545+0000","mapping_epoch":139,"log_start":"159'9500","ondisk_log_start":"159'9500","created":139,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:42:06.761192+0000","last_clean_scrub_stamp":"2024-09-13T14:42:06.761192+0000","objects_scrubbed":0,"log_size":6329,"log_dups_size":0,"ondisk_log_size":6329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:18.000113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15829,"num_write_kb":521088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":201,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975459+0000","last_change":"2024-09-13T14:35:58.853459+0000","last_active":"2024-09-13T14:45:49.975459+0000","last_peered":"2024-09-13T14:45:49.975459+0000","last_clean":"2024-09-13T14:45:49.975459+0000","last_became_active":"2024-09-13T14:35:58.852847+0000","last_became_peered":"2024-09-13T14:35:58.852847+0000","last_unstale":"2024-09-13T14:45:49.975459+0000","last_undegraded":"2024-09-13T14:45:49.975459+0000","last_fullsized":"2024-09-13T14:45:49.975459+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:19.067583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":154,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:30.908088+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:45:30.908088+0000","last_peered":"2024-09-13T14:45:30.908088+0000","last_clean":"2024-09-13T14:45:30.908088+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:45:30.908088+0000","last_undegraded":"2024-09-13T14:45:30.908088+0000","last_fullsized":"2024-09-13T14:45:30.908088+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":126,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:41.345601+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:45:41.345601+0000","last_peered":"2024-09-13T14:45:41.345601+0000","last_clean":"2024-09-13T14:45:41.345601+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:45:41.345601+0000","last_undegraded":"2024-09-13T14:45:41.345601+0000","last_fullsized":"2024-09-13T14:45:41.345601+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019470+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:45:29.019470+0000","last_peered":"2024-09-13T14:45:29.019470+0000","last_clean":"2024-09-13T14:45:29.019470+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:45:29.019470+0000","last_undegraded":"2024-09-13T14:45:29.019470+0000","last_fullsized":"2024-09-13T14:45:29.019470+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":148,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019854+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:45:29.019854+0000","last_peered":"2024-09-13T14:45:29.019854+0000","last_clean":"2024-09-13T14:45:29.019854+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:45:29.019854+0000","last_undegraded":"2024-09-13T14:45:29.019854+0000","last_fullsized":"2024-09-13T14:45:29.019854+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":87,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.960343+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:45:49.960343+0000","last_peered":"2024-09-13T14:45:49.960343+0000","last_clean":"2024-09-13T14:45:49.960343+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:45:49.960343+0000","last_undegraded":"2024-09-13T14:45:49.960343+0000","last_fullsized":"2024-09-13T14:45:49.960343+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":113,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.954946+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:45:49.954946+0000","last_peered":"2024-09-13T14:45:49.954946+0000","last_clean":"2024-09-13T14:45:49.954946+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:45:49.954946+0000","last_undegraded":"2024-09-13T14:45:49.954946+0000","last_fullsized":"2024-09-13T14:45:49.954946+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":139,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019540+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:45:29.019540+0000","last_peered":"2024-09-13T14:45:29.019540+0000","last_clean":"2024-09-13T14:45:29.019540+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:45:29.019540+0000","last_undegraded":"2024-09-13T14:45:29.019540+0000","last_fullsized":"2024-09-13T14:45:29.019540+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.957656+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:45:49.957656+0000","last_peered":"2024-09-13T14:45:49.957656+0000","last_clean":"2024-09-13T14:45:49.957656+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:45:49.957656+0000","last_undegraded":"2024-09-13T14:45:49.957656+0000","last_fullsized":"2024-09-13T14:45:49.957656+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":195,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975845+0000","last_change":"2024-09-13T14:40:37.189640+0000","last_active":"2024-09-13T14:45:49.975845+0000","last_peered":"2024-09-13T14:45:49.975845+0000","last_clean":"2024-09-13T14:45:49.975845+0000","last_became_active":"2024-09-13T14:40:37.189226+0000","last_became_peered":"2024-09-13T14:40:37.189226+0000","last_unstale":"2024-09-13T14:45:49.975845+0000","last_undegraded":"2024-09-13T14:45:49.975845+0000","last_fullsized":"2024-09-13T14:45:49.975845+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:51.633466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.968991+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:45:49.968991+0000","last_peered":"2024-09-13T14:45:49.968991+0000","last_clean":"2024-09-13T14:45:49.968991+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:45:49.968991+0000","last_undegraded":"2024-09-13T14:45:49.968991+0000","last_fullsized":"2024-09-13T14:45:49.968991+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:40.909531+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:45:40.909531+0000","last_peered":"2024-09-13T14:45:40.909531+0000","last_clean":"2024-09-13T14:45:40.909531+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:45:40.909531+0000","last_undegraded":"2024-09-13T14:45:40.909531+0000","last_fullsized":"2024-09-13T14:45:40.909531+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.969237+0000","last_change":"2024-09-13T14:40:37.184155+0000","last_active":"2024-09-13T14:45:49.969237+0000","last_peered":"2024-09-13T14:45:49.969237+0000","last_clean":"2024-09-13T14:45:49.969237+0000","last_became_active":"2024-09-13T14:40:37.183122+0000","last_became_peered":"2024-09-13T14:40:37.183122+0000","last_unstale":"2024-09-13T14:45:49.969237+0000","last_undegraded":"2024-09-13T14:45:49.969237+0000","last_fullsized":"2024-09-13T14:45:49.969237+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:37.148906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":198,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975285+0000","last_change":"2024-09-13T14:40:37.188438+0000","last_active":"2024-09-13T14:45:49.975285+0000","last_peered":"2024-09-13T14:45:49.975285+0000","last_clean":"2024-09-13T14:45:49.975285+0000","last_became_active":"2024-09-13T14:40:37.188096+0000","last_became_peered":"2024-09-13T14:40:37.188096+0000","last_unstale":"2024-09-13T14:45:49.975285+0000","last_undegraded":"2024-09-13T14:45:49.975285+0000","last_fullsized":"2024-09-13T14:45:49.975285+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:01:21.910423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":166,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959143+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:45:49.959143+0000","last_peered":"2024-09-13T14:45:49.959143+0000","last_clean":"2024-09-13T14:45:49.959143+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:45:49.959143+0000","last_undegraded":"2024-09-13T14:45:49.959143+0000","last_fullsized":"2024-09-13T14:45:49.959143+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":168,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.964397+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:45:49.964397+0000","last_peered":"2024-09-13T14:45:49.964397+0000","last_clean":"2024-09-13T14:45:49.964397+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:45:49.964397+0000","last_undegraded":"2024-09-13T14:45:49.964397+0000","last_fullsized":"2024-09-13T14:45:49.964397+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":92,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.957786+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:45:49.957786+0000","last_peered":"2024-09-13T14:45:49.957786+0000","last_clean":"2024-09-13T14:45:49.957786+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:45:49.957786+0000","last_undegraded":"2024-09-13T14:45:49.957786+0000","last_fullsized":"2024-09-13T14:45:49.957786+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":193,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.954690+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:45:49.954690+0000","last_peered":"2024-09-13T14:45:49.954690+0000","last_clean":"2024-09-13T14:45:49.954690+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:45:49.954690+0000","last_undegraded":"2024-09-13T14:45:49.954690+0000","last_fullsized":"2024-09-13T14:45:49.954690+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":201,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.982253+0000","last_change":"2024-09-13T14:40:38.176490+0000","last_active":"2024-09-13T14:45:49.982253+0000","last_peered":"2024-09-13T14:45:49.982253+0000","last_clean":"2024-09-13T14:45:49.982253+0000","last_became_active":"2024-09-13T14:40:38.176035+0000","last_became_peered":"2024-09-13T14:40:38.176035+0000","last_unstale":"2024-09-13T14:45:49.982253+0000","last_undegraded":"2024-09-13T14:45:49.982253+0000","last_fullsized":"2024-09-13T14:45:49.982253+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:08:46.807360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":154,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959320+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:45:49.959320+0000","last_peered":"2024-09-13T14:45:49.959320+0000","last_clean":"2024-09-13T14:45:49.959320+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:45:49.959320+0000","last_undegraded":"2024-09-13T14:45:49.959320+0000","last_fullsized":"2024-09-13T14:45:49.959320+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":141,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.962603+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:45:49.962603+0000","last_peered":"2024-09-13T14:45:49.962603+0000","last_clean":"2024-09-13T14:45:49.962603+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:45:49.962603+0000","last_undegraded":"2024-09-13T14:45:49.962603+0000","last_fullsized":"2024-09-13T14:45:49.962603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":154,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.959152+0000","last_change":"2024-09-13T14:40:37.187568+0000","last_active":"2024-09-13T14:45:49.959152+0000","last_peered":"2024-09-13T14:45:49.959152+0000","last_clean":"2024-09-13T14:45:49.959152+0000","last_became_active":"2024-09-13T14:40:37.187263+0000","last_became_peered":"2024-09-13T14:40:37.187263+0000","last_unstale":"2024-09-13T14:45:49.959152+0000","last_undegraded":"2024-09-13T14:45:49.959152+0000","last_fullsized":"2024-09-13T14:45:49.959152+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:34:50.496845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":126,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.962818+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:45:49.962818+0000","last_peered":"2024-09-13T14:45:49.962818+0000","last_clean":"2024-09-13T14:45:49.962818+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:45:49.962818+0000","last_undegraded":"2024-09-13T14:45:49.962818+0000","last_fullsized":"2024-09-13T14:45:49.962818+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":200,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.957958+0000","last_change":"2024-09-13T14:40:37.184965+0000","last_active":"2024-09-13T14:45:49.957958+0000","last_peered":"2024-09-13T14:45:49.957958+0000","last_clean":"2024-09-13T14:45:49.957958+0000","last_became_active":"2024-09-13T14:40:37.184598+0000","last_became_peered":"2024-09-13T14:40:37.184598+0000","last_unstale":"2024-09-13T14:45:49.957958+0000","last_undegraded":"2024-09-13T14:45:49.957958+0000","last_fullsized":"2024-09-13T14:45:49.957958+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:07.001741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":189,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.958131+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:45:49.958131+0000","last_peered":"2024-09-13T14:45:49.958131+0000","last_clean":"2024-09-13T14:45:49.958131+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:45:49.958131+0000","last_undegraded":"2024-09-13T14:45:49.958131+0000","last_fullsized":"2024-09-13T14:45:49.958131+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":173,"state":"active+clean","last_fresh":"2024-09-13T14:45:29.019089+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:45:29.019089+0000","last_peered":"2024-09-13T14:45:29.019089+0000","last_clean":"2024-09-13T14:45:29.019089+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:45:29.019089+0000","last_undegraded":"2024-09-13T14:45:29.019089+0000","last_fullsized":"2024-09-13T14:45:29.019089+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":202,"reported_epoch":174,"state":"active+clean","last_fresh":"2024-09-13T14:45:49.975584+0000","last_change":"2024-09-13T14:40:02.368959+0000","last_active":"2024-09-13T14:45:49.975584+0000","last_peered":"2024-09-13T14:45:49.975584+0000","last_clean":"2024-09-13T14:45:49.975584+0000","last_became_active":"2024-09-13T14:40:02.368590+0000","last_became_peered":"2024-09-13T14:40:02.368590+0000","last_unstale":"2024-09-13T14:45:49.975584+0000","last_undegraded":"2024-09-13T14:45:49.975584+0000","last_fullsized":"2024-09-13T14:45:49.975584+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:38:54.736815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":490471448,"num_objects":7485,"num_object_clones":0,"num_object_copies":22455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7485,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":252409,"num_write_kb":8316545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1403990016,"data_stored":1403977800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124268,"ondisk_log_size":124268,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234310,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":947940,"kb_used_data":271068,"kb_used_omap":1,"kb_used_meta":676734,"kb_avail":92797212,"statfs":{"total":95995035648,"available":95024345088,"internally_reserved":0,"allocated":277573632,"data_stored":276706746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":692976075},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495948,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":634908,"kb_used_data":94356,"kb_used_omap":1,"kb_used_meta":540542,"kb_avail":93110244,"statfs":{"total":95995035648,"available":95344889856,"internally_reserved":0,"allocated":96620544,"data_stored":95761789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":553515463},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757586,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1082648,"kb_used_data":297616,"kb_used_omap":1,"kb_used_meta":785022,"kb_avail":92662504,"statfs":{"total":95995035648,"available":94886404096,"internally_reserved":0,"allocated":304758784,"data_stored":303900188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":803862982},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019222,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":702748,"kb_used_data":90004,"kb_used_omap":1,"kb_used_meta":612734,"kb_avail":93042404,"statfs":{"total":95995035648,"available":95275421696,"internally_reserved":0,"allocated":92164096,"data_stored":91305364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":627440070},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650434,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494416,"kb_used_data":278280,"kb_used_omap":4,"kb_used_meta":216059,"kb_avail":93250736,"statfs":{"total":95995035648,"available":95488753664,"internally_reserved":0,"allocated":284958720,"data_stored":284108054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":221244975},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879369,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219288,"kb_used_data":103760,"kb_used_omap":6,"kb_used_meta":115513,"kb_avail":93525864,"statfs":{"total":95995035648,"available":95770484736,"internally_reserved":0,"allocated":106250240,"data_stored":105391998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":118285881},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435386,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":590476,"kb_used_data":201540,"kb_used_omap":5,"kb_used_meta":388922,"kb_avail":93154676,"statfs":{"total":95995035648,"available":95390388224,"internally_reserved":0,"allocated":206376960,"data_stored":205529287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":398256481},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925960,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":458000,"kb_used_data":159880,"kb_used_omap":5,"kb_used_meta":298106,"kb_avail":93287152,"statfs":{"total":95995035648,"available":95526043648,"internally_reserved":0,"allocated":163717120,"data_stored":162866424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":305260714},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":157483008,"data_stored":157483008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":201592832,"data_stored":201588760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":98893824,"data_stored":98893824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":252051456,"data_stored":252051456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":82903040,"data_stored":82903040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":271716352,"data_stored":271712280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":92209152,"data_stored":92209152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":247140352,"data_stored":247136280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:46:00.372 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:46:00.372 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:46:00.372 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:00.372 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd out 0 2024-09-13T14:46:00.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:00.482 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:00.584 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:00.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:00.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:00.744 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:00.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:00.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:00.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:00.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:00.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:01.124 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:01.166 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:01.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:01.268 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.820 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:01.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:01.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:01.990 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:01.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:02.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:01 smithi042 ceph-mon[113926]: pgmap v622: 99 pgs: 99 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:46:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:01 smithi042 ceph-mon[108570]: pgmap v622: 99 pgs: 99 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:46:02.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:02.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:01 smithi159 ceph-mon[102519]: pgmap v622: 99 pgs: 99 active+clean; 414 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T14:46:02.259 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:02.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:02.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.400 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:02.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:02.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:02.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:02.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:02.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:02.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:02.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:02.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:02.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:02.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:02.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:03.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:03.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:03.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:03.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:03.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:03.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:03.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:03.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:03.501 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:03.603 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:03.636 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:03.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:03.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 129946 objects 2024-09-13T14:46:03.753 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :137.156 2024-09-13T14:46:03.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:03.838 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:03.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:03.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:03.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:03.992 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:03 smithi042 ceph-mon[108570]: pgmap v623: 99 pgs: 99 active+clean; 228 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:46:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:03 smithi042 ceph-mon[113926]: pgmap v623: 99 pgs: 99 active+clean; 228 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:46:04.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:04.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:04.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:03 smithi159 ceph-mon[102519]: pgmap v623: 99 pgs: 99 active+clean; 228 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-13T14:46:04.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:04.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:04.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:04.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:04.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:04.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:04.520 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:46:04.521 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:04.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:04.537 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:46:04.537 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:04.553 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:46:04.553 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:04.570 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:46:04.570 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:04.587 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:46:04.587 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:04.604 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:46:04.604 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:04.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:46:04.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:04.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:04.638 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:46:04.638 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:04.646 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:04.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:04.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:04.871 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:04.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:04.896 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:04.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:04.997 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:05.022 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:05.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:05.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:05.149 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:05.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2915476620' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:05.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:05.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:05.337 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2915476620' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:05.337 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2915476620' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:05.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:05.400 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:05.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:05.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:05.524 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:05.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:05.626 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:05.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:05.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:05.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:05.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:05.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:05.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:05.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:05.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:06.008 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:06.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:06.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:06.158 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:06.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:05 smithi159 ceph-mon[102519]: pgmap v624: 99 pgs: 99 active+clean; 115 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:46:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:06.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:05 smithi042 ceph-mon[108570]: pgmap v624: 99 pgs: 99 active+clean; 115 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:46:06.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:06.252 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:05 smithi042 ceph-mon[113926]: pgmap v624: 99 pgs: 99 active+clean; 115 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T14:46:06.252 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:06.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:06.284 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:06.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:06.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:06.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:06.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:06.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:06.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:06.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:06.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:06.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:06.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:06.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:06.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:07.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:07.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:07.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:07.158 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:07.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:07.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:07.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:07.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:07.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:07.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:07.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:07.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:07.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:07.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:07.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:07.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.006 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:46:08.008 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:08.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:08.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:08.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:08.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:08.225 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-13T14:46:08.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-13T14:46:08.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[113926]: pgmap v625: 99 pgs: 99 active+clean; 17 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:46:08.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[108570]: pgmap v625: 99 pgs: 99 active+clean; 17 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.251 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:07 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.268 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:08.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.416 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:07 smithi159 ceph-mon[102519]: pgmap v625: 99 pgs: 99 active+clean; 17 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2024-09-13T14:46:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:07 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:07 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:08.517 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.539 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:08.600 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:121416 8.a 8:521c73d4:::benchmark_data_smithi042_130700_object121415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 569.32936977899999, 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.008701308, 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.601 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 121416 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240586+0000", 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240588+0000", 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2309999999999999e-06 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240617+0000", 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9275e-05 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240638+0000", 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1373e-05 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240650+0000", 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1075e-05 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240760+0000", 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000110196 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.240772+0000", 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1838000000000001e-05 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.231939+0000", 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99116713999999995 2024-09-13T14:46:08.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.232079+0000", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013983299999999999 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.232195+0000", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116772 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.238677+0000", 2024-09-13T14:46:08.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064812339999999998 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.243060+0000", 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043835360000000004 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249201+0000", 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061413140000000001 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249238+0000", 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7011000000000002e-05 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249287+0000", 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8479999999999997e-05 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:121422 8.a 8:52568a6a:::benchmark_data_smithi042_130700_object121421:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e112)", 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 569.31935305499997, 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0054267990000001, 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 121422 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250602+0000", 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250604+0000", 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.72e-06 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250630+0000", 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6095e-05 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250645+0000", 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4562999999999999e-05 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.250654+0000", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3559999999999999e-06 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.254323+0000", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003669285 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:39.254334+0000", 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0791999999999999e-05 2024-09-13T14:46:08.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234556+0000", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98022220000000004 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234637+0000", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1094000000000007e-05 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.234736+0000", 2024-09-13T14:46:08.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8134999999999995e-05 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.243017+0000", 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082817710000000003 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.249358+0000", 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0063408120000000004 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.255957+0000", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0065990629999999996 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.255995+0000", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7732999999999999e-05 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:36:40.256029+0000", 2024-09-13T14:46:08.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4180000000000001e-05 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187657 8.3 8:c5d07f4c:::benchmark_data_smithi042_130700_object63068:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.70692407199999, 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0351328040000001, 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:46:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187657 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863031+0000", 2024-09-13T14:46:08.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863032+0000", 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0920000000000001e-06 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863036+0000", 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.946e-06 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863059+0000", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2549e-05 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863068+0000", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1050000000000001e-06 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864184+0000", 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001115935 2024-09-13T14:46:08.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864193+0000", 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9369999999999993e-06 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898090+0000", 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0338969140000001 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898164+0000", 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4326000000000006e-05 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187670 8.3 8:c61da759:::benchmark_data_smithi042_130700_object63081:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.706875094, 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.035638155, 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.621 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187670 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863080+0000", 2024-09-13T14:46:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863081+0000", 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1200000000000002e-07 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863084+0000", 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5329999999999999e-06 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863092+0000", 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7710000000000006e-06 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.863098+0000", 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3730000000000001e-06 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864683+0000", 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001584963 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864691+0000", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1519999999999992e-06 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898675+0000", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.03398452 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898718+0000", 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3130999999999998e-05 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.625 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187680 8.3 8:ce9e033c:::benchmark_data_smithi042_130700_object63091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.70539513699998, 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0633122770000001, 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.626 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187680 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864560+0000", 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864562+0000", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4980000000000001e-06 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864566+0000", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4660000000000001e-06 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864586+0000", 2024-09-13T14:46:08.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9403e-05 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.864597+0000", 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1782e-05 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.865289+0000", 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000691923 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.865320+0000", 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0431000000000001e-05 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898763+0000", 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0334432250000001 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.898897+0000", 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013364799999999999 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.899526+0000", 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00062976899999999997 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924433+0000", 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.024906927999999998 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927694+0000", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003260256 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927742+0000", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8282999999999998e-05 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927815+0000", 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2917000000000007e-05 2024-09-13T14:46:08.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927872+0000", 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7747999999999997e-05 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187708 8.3 8:cf4a0971:::benchmark_data_smithi042_130700_object63119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.66586810400003, 2024-09-13T14:46:08.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.023983487, 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187708 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:46:08.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904087+0000", 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904088+0000", 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2720000000000001e-06 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904093+0000", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1350000000000002e-06 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904102+0000", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6299999999999993e-06 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.904108+0000", 2024-09-13T14:46:08.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1820000000000003e-06 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.905077+0000", 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00096839400000000003 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.905088+0000", 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1216000000000001e-05 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918236+0000", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0131478519999999 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918369+0000", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013295000000000001 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.918512+0000", 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014321000000000001 2024-09-13T14:46:08.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927703+0000", 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0091908170000000004 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927937+0000", 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023419900000000001 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927983+0000", 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5988000000000001e-05 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928034+0000", 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0537000000000001e-05 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928071+0000", 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7104999999999998e-05 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.640 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187719 8.3 8:cec909fa:::benchmark_data_smithi042_130700_object63130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.65362336999999, 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0193972339999999, 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.641 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187719 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916332+0000", 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916334+0000", 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.818e-06 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916342+0000", 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9219999999999997e-06 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916360+0000", 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8223000000000002e-05 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916370+0000", 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0088999999999999e-05 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916613+0000", 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024308700000000001 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.916624+0000", 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1252e-05 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921255+0000", 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0046310679999999 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921382+0000", 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012638100000000001 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.921523+0000", 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014164099999999999 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.927947+0000", 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064236600000000003 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928145+0000", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019809999999999999 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935622+0000", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0074763360000000001 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935681+0000", 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9567000000000001e-05 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935729+0000", 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8090000000000002e-05 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187733 8.3 8:cb6a9569:::benchmark_data_smithi042_130700_object63144:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.63778486199999, 2024-09-13T14:46:08.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0037710639999999, 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187733 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:46:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932170+0000", 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932172+0000", 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932178+0000", 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2469999999999997e-06 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932193+0000", 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4273999999999999e-05 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932201+0000", 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6710000000000005e-06 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932268+0000", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6944000000000001e-05 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.932278+0000", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5559999999999996e-06 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924468+0000", 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99218998800000002 2024-09-13T14:46:08.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924562+0000", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4061999999999995e-05 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.924703+0000", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014133599999999999 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928170+0000", 2024-09-13T14:46:08.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034668429999999998 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935774+0000", 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076034570000000001 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935825+0000", 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1285000000000001e-05 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935900+0000", 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5479e-05 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935942+0000", 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1179e-05 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.655 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:187739 8.3 8:c5e9808e:::benchmark_data_smithi042_130700_object63150:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 484.628590288, 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99483940699999995, 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 187739 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941365+0000", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941367+0000", 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.702e-06 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941373+0000", 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7309999999999996e-06 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941387+0000", 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3618000000000001e-05 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941396+0000", 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5590000000000006e-06 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941466+0000", 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0840000000000006e-05 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:03.941478+0000", 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1802e-05 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928378+0000", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98689947099999997 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928464+0000", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6094000000000006e-05 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,6", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.928566+0000", 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010249 2024-09-13T14:46:08.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935785+0000", 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072182069999999999 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.935993+0000", 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020831300000000001 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.661 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936117+0000", 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012453799999999999 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936169+0000", 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1196000000000002e-05 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:04.936204+0000", 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5846000000000002e-05 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225301 8.1 8:816db9f2:::benchmark_data_smithi042_130700_object100712:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 435.76597829000002, 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.055774234, 2024-09-13T14:46:08.663 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225301 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803977+0000", 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803978+0000", 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4339999999999999e-06 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803984+0000", 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8019999999999997e-06 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.803997+0000", 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3149999999999999e-05 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804007+0000", 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2939999999999994e-06 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804075+0000", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8554999999999997e-05 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op must wait for map", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.804083+0000", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6650000000000003e-06 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807396+0000", 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003313561 2024-09-13T14:46:08.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807405+0000", 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0669999999999992e-06 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.859673+0000", 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0522687049999999 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.859751+0000", 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8001000000000001e-05 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225319 8.1 8:8a6235bd:::benchmark_data_smithi042_130700_object100730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 435.76479289700001, 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0854911300000001, 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.669 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225319 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805162+0000", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805163+0000", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0279999999999999e-06 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805168+0000", 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3270000000000002e-06 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805178+0000", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0237e-05 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805185+0000", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9480000000000002e-06 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805235+0000", 2024-09-13T14:46:08.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9752000000000001e-05 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting_for_map not empty", 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.805241+0000", 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9279999999999998e-06 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807424+0000", 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021835499999999998 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.807430+0000", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4399999999999996e-06 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860239+0000", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0528097700000001 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860357+0000", 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117201 2024-09-13T14:46:08.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.860943+0000", 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00058651199999999997 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882766+0000", 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021822478999999999 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882837+0000", 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1075000000000006e-05 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890439+0000", 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076019579999999998 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890592+0000", 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015348400000000001 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890654+0000", 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1440999999999997e-05 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225326 8.1 8:8dc278eb:::benchmark_data_smithi042_130700_object100737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 435.75315820399999, 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.07410107, 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.677 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225326 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816797+0000", 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816798+0000", 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0699999999999997e-07 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.679 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816800+0000", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.176e-06 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816806+0000", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.421e-06 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816811+0000", 2024-09-13T14:46:08.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7639999999999996e-06 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816908+0000", 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6739999999999999e-05 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.816918+0000", 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0036e-05 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876442+0000", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.059524414 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876582+0000", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000139326 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.876725+0000", 2024-09-13T14:46:08.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000143304 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890664+0000", 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013939058000000001 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890706+0000", 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2361999999999999e-05 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890766+0000", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9821000000000002e-05 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890852+0000", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6096999999999999e-05 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890898+0000", 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6044000000000002e-05 2024-09-13T14:46:08.684 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225356 8.1 8:8e84ccef:::benchmark_data_smithi042_130700_object100767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 435.720345501, 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0491585130000001, 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.685 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225356 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849610+0000", 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849611+0000", 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.378e-06 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849617+0000", 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2989999999999996e-06 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849628+0000", 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1215e-05 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849635+0000", 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3880000000000002e-06 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849696+0000", 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1148000000000005e-05 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.849706+0000", 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8120000000000002e-06 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879680+0000", 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0299740319999999 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879768+0000", 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7418999999999998e-05 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.879844+0000", 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6153000000000001e-05 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890717+0000", 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010873551 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.690 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898610+0000", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078932029999999997 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898661+0000", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0658999999999997e-05 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898717+0000", 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6248000000000001e-05 2024-09-13T14:46:08.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898768+0000", 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1007999999999997e-05 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34323.0:225369 8.1 8:82518930:::benchmark_data_smithi042_130700_object100780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:46:08.692 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 435.70168809699999, 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0307719120000001, 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34323", 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/1585056441", 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 225369 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.693 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868267+0000", 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868269+0000", 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6029999999999999e-06 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868275+0000", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7640000000000002e-06 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868298+0000", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3754e-05 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868306+0000", 2024-09-13T14:46:08.695 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7260000000000002e-06 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868365+0000", 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8529999999999997e-05 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:52.868374+0000", 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5589999999999994e-06 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882877+0000", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0145028629999999 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.882967+0000", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0200999999999999e-05 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,2", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.883039+0000", 2024-09-13T14:46:08.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1970000000000004e-05 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.890722+0000", 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076824600000000003 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898810+0000", 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0080881570000000003 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898866+0000", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5825000000000002e-05 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.898993+0000", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012693399999999999 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:38:53.899039+0000", 2024-09-13T14:46:08.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6566000000000001e-05 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119542 9.d 9:ba7fe936:::benchmark_data_smithi042_157727_object119541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:34.919378+0000", 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 153.650577153, 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99447441400000003, 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:08.700 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119542 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919378+0000", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919378+0000", 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:08.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919380+0000", 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.517e-06 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919403+0000", 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3184e-05 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:08.702 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919417+0000", 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4504999999999999e-05 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.919426+0000", 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8920000000000006e-06 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.920181+0000", 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00075459799999999999 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.920191+0000", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0224000000000001e-05 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.912169+0000", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99197781299999999 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.913853+0000", 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001683681 2024-09-13T14:46:08.704 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.705 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.705 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.705 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:08.705 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:08.705 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:46:08.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:08.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:08.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.852 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:08.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:08.953 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:08.976 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:08.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:09.077 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:09.099 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:09.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:09.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:09.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:09.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:09.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:09.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:09.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:09.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:09.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:09.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3800444142' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: osdmap e175: 8 total, 8 up, 8 in 2024-09-13T14:46:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:46:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:46:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:09.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:09.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:09.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:09.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.724 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:09.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:09.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:09] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.51.0" 2024-09-13T14:46:09.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:09.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:09.976 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:09.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:10.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.104 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:10.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:10.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:10.274 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:10 smithi042 ceph-mon[113926]: pgmap v627: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:46:10.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:10 smithi042 ceph-mon[108570]: pgmap v627: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:46:10.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:46:10.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:10.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:10.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:10.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:10.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:10.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:10.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:10 smithi159 ceph-mon[102519]: pgmap v627: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 748 op/s 2024-09-13T14:46:10.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:10.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:10.736 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:10.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:10.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:10.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:10.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:10.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:10.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:10.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:11.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:11.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:11.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:11.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:11.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:11.259 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:11.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:11.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:11.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:11.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:11.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:11.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:11.617 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:11.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:11.675 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:11 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T14:46:11.421+0000 7fcad2a32640 -1 mon.a@0(leader).osd e175 definitely_dead 0 2024-09-13T14:46:11.718 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:11.741 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:11.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:11.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:11.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:11.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:11.990 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:11.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:11.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:11.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.022 INFO:teuthology.orchestra.run.smithi042.stderr:marked out osd.0. 2024-09-13T14:46:12.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:12.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:12.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:12.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:12.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:12.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[113926]: pgmap v628: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:46:12.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3304302436' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[113926]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:46:12.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[108570]: pgmap v628: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:46:12.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3304302436' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:12 smithi042 ceph-mon[108570]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:46:12.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:12.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:12.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:12.498 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:12 smithi159 ceph-mon[102519]: pgmap v628: 99 pgs: 99 active+clean; 582 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 660 op/s 2024-09-13T14:46:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3304302436' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:12 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T14:46:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:12 smithi159 ceph-mon[102519]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T14:46:12.600 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:12.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:12.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:12.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.754 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:12.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:12.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:12.879 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:12.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:12.979 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:13.001 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:13.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:46:13.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:13.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:13.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:13.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:13.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:13 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:46:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:13 smithi042 ceph-mon[113926]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:13 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:46:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:13 smithi042 ceph-mon[108570]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:13.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:13.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.377 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:13.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:13.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:13 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T14:46:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:13 smithi159 ceph-mon[102519]: osdmap e176: 8 total, 8 up, 7 in 2024-09-13T14:46:13.503 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:13.629 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:13.756 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:13.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:13.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:13.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:13.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:13.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:14.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:14.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:14.290 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:14 smithi042 ceph-mon[113926]: pgmap v630: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 229 op/s 2024-09-13T14:46:14.290 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:14 smithi042 ceph-mon[113926]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:14.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:14 smithi042 ceph-mon[108570]: pgmap v630: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 229 op/s 2024-09-13T14:46:14.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:14 smithi042 ceph-mon[108570]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:14.312 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:14.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:14 smithi159 ceph-mon[102519]: pgmap v630: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 229 op/s 2024-09-13T14:46:14.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:14 smithi159 ceph-mon[102519]: osdmap e177: 8 total, 8 up, 7 in 2024-09-13T14:46:14.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:14.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:14.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:14.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:14.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:14.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:14.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:14.683 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:14.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:14.785 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:14.809 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:14.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:14.927 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:14.928 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:14.929 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:14.929 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:14.929 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:14.933 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:14.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:14.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:15.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:15.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:15.135 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:15.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:15.185 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:15.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:15.222 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:15.222 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 174'16370 2024-09-13T14:46:15.222 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 174'16088 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 174'16216 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 174'16336 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1026166798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 174'16370 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 174'16088 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 174'16216 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 174'16336 2024-09-13T14:46:15.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1026166798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:15.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:15.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:15.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:15.410 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:15.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: osdmap e178: 8 total, 8 up, 7 in 2024-09-13T14:46:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 174'16370 2024-09-13T14:46:15.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: 9.9 starting backfill to osd.3 from (0'0,0'0] MAX to 174'16088 2024-09-13T14:46:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 174'16216 2024-09-13T14:46:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: 9.b starting backfill to osd.2 from (0'0,0'0] MAX to 174'16336 2024-09-13T14:46:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1026166798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:15.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:15.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.555 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:15.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:15.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:15.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:15.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:15.837 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:15.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:15.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:15.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:16.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:16.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:16.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:16.188 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:16.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:16.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:16.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:16 smithi042 ceph-mon[108570]: pgmap v633: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:46:16.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:16 smithi042 ceph-mon[108570]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:16.324 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:16 smithi042 ceph-mon[113926]: pgmap v633: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:46:16.324 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:16 smithi042 ceph-mon[113926]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:16.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:16.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:16.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:16.463 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:16.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:16.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:16 smithi159 ceph-mon[102519]: pgmap v633: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T14:46:16.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:16 smithi159 ceph-mon[102519]: osdmap e179: 8 total, 8 up, 7 in 2024-09-13T14:46:16.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:16.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:16.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:16.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:16.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:16.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:16.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:16.879 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:16.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:16.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:16.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:17.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:17.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:17.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:17.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:17.255 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:17 smithi042 ceph-mon[108570]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:17.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:17 smithi042 ceph-mon[113926]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:17.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:17.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:17.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:17.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:17.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:17.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:17 smithi159 ceph-mon[102519]: osdmap e180: 8 total, 8 up, 7 in 2024-09-13T14:46:17.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:17.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:17.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:17.686 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:17.687 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:17.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:17.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:17.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:17.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:17.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:17.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:17.963 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:17.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:17.978 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:46:17.979 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:46:18.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:18.087 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:18.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:18.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:18.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:18.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:18 smithi042 ceph-mon[108570]: pgmap v636: 99 pgs: 4 peering, 95 active+clean; 634 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 2 objects/s recovering 2024-09-13T14:46:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:18 smithi042 ceph-mon[113926]: pgmap v636: 99 pgs: 4 peering, 95 active+clean; 634 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 2 objects/s recovering 2024-09-13T14:46:18.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:18.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:18.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:18 smithi159 ceph-mon[102519]: pgmap v636: 99 pgs: 4 peering, 95 active+clean; 634 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 2 objects/s recovering 2024-09-13T14:46:18.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:18.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:18.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:18.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:18.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:18.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:18.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:18.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:18.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:18.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:18.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:19.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:19.096 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:19.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:19.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:19.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:19.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:19.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:19.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:19.454 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:19.480 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:19.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:19.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:46:19.582 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:19.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:19.709 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:19.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:19.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:19.837 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:19.860 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:19.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:19.984 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:19.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:20.047 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:19] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.51.0" 2024-09-13T14:46:20.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:20.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:20.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:20.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:20.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:20.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:20 smithi042 ceph-mon[113926]: pgmap v637: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 29 KiB/s rd, 84 KiB/s wr, 20 op/s; 60 KiB/s, 2 objects/s recovering 2024-09-13T14:46:20.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:20 smithi042 ceph-mon[108570]: pgmap v637: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 29 KiB/s rd, 84 KiB/s wr, 20 op/s; 60 KiB/s, 2 objects/s recovering 2024-09-13T14:46:20.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:20.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:20.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:20.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:20.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:20.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:20 smithi159 ceph-mon[102519]: pgmap v637: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 29 KiB/s rd, 84 KiB/s wr, 20 op/s; 60 KiB/s, 2 objects/s recovering 2024-09-13T14:46:20.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:20.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:20.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:20.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:20.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.741 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:20.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:20.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:20.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:20.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:20.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:20.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:20.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:20.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:21.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.124 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:21.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:21.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:21.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.352 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:21.362 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:21.377 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:21.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:21.479 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:21.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:21.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:21.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:21.630 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:21.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:21.732 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:21.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:21.859 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:21.884 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:21.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:21.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:22.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:22.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:22.055 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:46:22.070 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:22.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:22.133 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:22.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:22.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:22 smithi042 ceph-mon[108570]: pgmap v638: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24 KiB/s rd, 70 KiB/s wr, 17 op/s; 50 KiB/s, 1 objects/s recovering 2024-09-13T14:46:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:22 smithi042 ceph-mon[113926]: pgmap v638: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24 KiB/s rd, 70 KiB/s wr, 17 op/s; 50 KiB/s, 1 objects/s recovering 2024-09-13T14:46:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:22.362 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:22.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:22.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:22.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:22 smithi159 ceph-mon[102519]: pgmap v638: 99 pgs: 4 peering, 95 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24 KiB/s rd, 70 KiB/s wr, 17 op/s; 50 KiB/s, 1 objects/s recovering 2024-09-13T14:46:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:46:22.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:22.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:22.612 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:22.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:22.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:22.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.864 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:22.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:22.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:22.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:22.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:23.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:23.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:23.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:23.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:23.167 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:23.167 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:23.167 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[113926]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:23.167 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:23.167 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:23.168 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:23 smithi042 ceph-mon[108570]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:23.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:23.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:23.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:23.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:23.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:23.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:23 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:23 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3885683406' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:46:23.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:23 smithi159 ceph-mon[102519]: osdmap e181: 8 total, 8 up, 7 in 2024-09-13T14:46:23.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:23.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:23.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:23.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:23.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:23.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:23.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:23.890 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:23.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:23.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:24.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:24.071 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_4' removed 2024-09-13T14:46:24.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:24.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:24.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:24.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:24.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:24.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[108570]: pgmap v640: 99 pgs: 99 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 15 op/s; 44 KiB/s, 1 objects/s recovering 2024-09-13T14:46:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1084731790' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[113926]: pgmap v640: 99 pgs: 99 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 15 op/s; 44 KiB/s, 1 objects/s recovering 2024-09-13T14:46:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1084731790' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:24 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:24.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:24.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:24 smithi159 ceph-mon[102519]: pgmap v640: 99 pgs: 99 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 15 op/s; 44 KiB/s, 1 objects/s recovering 2024-09-13T14:46:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:24 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1084731790' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:24 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:46:24.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:24.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:24.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:24.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:24.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:24.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:24.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:24.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:24.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:24.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:24.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:25.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:25.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:25.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:25.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:25.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:25.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:25.232 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:46:25.232 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:25.243 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:46:25.243 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:25.259 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:46:25.260 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:25.276 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:46:25.276 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:25.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:25.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:46:25.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:25.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[113926]: 1.0 scrub starts 2024-09-13T14:46:25.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[113926]: 1.0 scrub ok 2024-09-13T14:46:25.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:46:25.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[113926]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:25.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[108570]: 1.0 scrub starts 2024-09-13T14:46:25.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[108570]: 1.0 scrub ok 2024-09-13T14:46:25.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:46:25.294 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:25 smithi042 ceph-mon[108570]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:25.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:25.309 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:46:25.310 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:25.326 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:46:25.327 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:25.343 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:46:25.343 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:25.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:25.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:25.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:25 smithi159 ceph-mon[102519]: 1.0 scrub starts 2024-09-13T14:46:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:25 smithi159 ceph-mon[102519]: 1.0 scrub ok 2024-09-13T14:46:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:25 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:46:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:25 smithi159 ceph-mon[102519]: osdmap e182: 8 total, 8 up, 7 in 2024-09-13T14:46:25.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:25.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:25.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:25.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:25.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:25.776 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:25.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:25.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:25.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:25.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:25.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:26.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:26.051 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:26.153 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:26.177 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:26.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:26.297 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:26 smithi042 ceph-mon[108570]: pgmap v642: 83 pgs: 83 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 9.4 KiB/s rd, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:46:26.297 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:26 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3822924579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.298 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:26 smithi042 ceph-mon[113926]: pgmap v642: 83 pgs: 83 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 9.4 KiB/s rd, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:46:26.298 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:26 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3822924579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:26.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:26.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:26.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:46:26.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:26 smithi159 ceph-mon[102519]: pgmap v642: 83 pgs: 83 active+clean; 687 KiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 9.4 KiB/s rd, 4 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:46:26.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:26 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3822924579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:26.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:26.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:26.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:26.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:26.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:26.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:26.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:26.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:26.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:27.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:27.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:27.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:27.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:27.271 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:27 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:46:27.271 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:27 smithi042 ceph-mon[108570]: 1.2 scrub ok 2024-09-13T14:46:27.271 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:27 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:46:27.271 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:27 smithi042 ceph-mon[113926]: 1.2 scrub ok 2024-09-13T14:46:27.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:27.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:27.409 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:27.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:27.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:27 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:46:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:27 smithi159 ceph-mon[102519]: 1.2 scrub ok 2024-09-13T14:46:27.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:27.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:27.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:27.689 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:27.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:27.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:27.803 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:46:27.804 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:46:27.804 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-13T14:46:27.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-13T14:46:27.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:27.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:27.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:27.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:27.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:28.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:28.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:28.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:28.199 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:28.241 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:46:28.241 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:46:28.241 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:46:28.241 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:46:28.241 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:46:28.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:28.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:28.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:28 smithi042 ceph-mon[108570]: pgmap v643: 83 pgs: 83 active+clean; 634 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:28.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:28 smithi042 ceph-mon[113926]: pgmap v643: 83 pgs: 83 active+clean; 634 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:28.428 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:28 smithi159 ceph-mon[102519]: pgmap v643: 83 pgs: 83 active+clean; 634 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:28.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:28.574 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:28.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.575 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:28.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:28.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:28.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:28.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:28.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:28.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:28.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:28.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:29.037 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.053 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:29.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:46:29.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.076 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:29.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:29.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:29.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:29.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:29.292 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:29.303 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:29.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:29.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:29.431 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:29.453 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:29.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:29.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:29.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:29.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:29.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:29.770 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.789 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:29.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:29.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:29.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:29.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:29] "GET /metrics HTTP/1.1" 200 36363 "" "Prometheus/2.51.0" 2024-09-13T14:46:29.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:29.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:29.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:29.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:30.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:30.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:30.229 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:30 smithi042 ceph-mon[108570]: pgmap v644: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:30.229 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:30 smithi042 ceph-mon[113926]: pgmap v644: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:30.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:30.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:30.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:30.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:30.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:30 smithi159 ceph-mon[102519]: pgmap v644: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:30.553 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:30.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:30.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:30.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:30.706 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:30.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:30.831 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:30.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:30.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:30.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:30.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:31.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:31.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:31.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:31.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:31.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:31.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:31.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:31.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:31.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:31.463 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:31.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:31.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:31.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:31.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:31.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:31.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:31.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:31.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:31.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:31.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:31.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:32.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:32.087 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:32.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:32 smithi042 ceph-mon[113926]: pgmap v645: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:32.153 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:32 smithi042 ceph-mon[108570]: pgmap v645: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:32.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:32.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:32.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:32.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:32.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:32.463 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:32.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:32 smithi159 ceph-mon[102519]: pgmap v645: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:32.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:32.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:32.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:32.693 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:32.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:32.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:32.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:32.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:32.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:32.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:32.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:33.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:33.095 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:33.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:33.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:33.221 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:33.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:33.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:33.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:33.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:33.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:33.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:33.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:33.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:33.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:33.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:33.721 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:33.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:33.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:33.950 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:33.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:33.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:34.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.162 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:34 smithi042 ceph-mon[113926]: pgmap v646: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:34.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:34 smithi042 ceph-mon[108570]: pgmap v646: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:34.203 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:34.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.225 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:34.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:34.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:34.351 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:34.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:34.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:34 smithi159 ceph-mon[102519]: pgmap v646: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:34.551 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:46:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:46:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:46:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:46:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:89170 9.0 e153/139 9:0c00bcda:::benchmark_data_smithi042_157727_object89169:head v 153'5528, mlcod=153'5526)", 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 208.85831847899999, 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.027526489000000001, 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:46:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614500+0000", 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614502+0000", 2024-09-13T14:46:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3060000000000001e-06 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614596+0000", 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4013999999999998e-05 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614636+0000", 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0667999999999998e-05 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614650+0000", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3679999999999999e-05 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614733+0000", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.276e-05 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.614763+0000", 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0431000000000001e-05 2024-09-13T14:46:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.624424+0000", 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096609380000000009 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:05.642027+0000", 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017602692 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111577 9.2 e155/139 9:458b1f83:::benchmark_data_smithi042_157727_object111576:head v 155'6951, mlcod=155'6948)", 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.708076033, 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36778615199999998, 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764743+0000", 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764744+0000", 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0240000000000001e-06 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764793+0000", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8990999999999999e-05 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764806+0000", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2989e-05 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764814+0000", 2024-09-13T14:46:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3100000000000001e-06 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764896+0000", 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2065999999999997e-05 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764923+0000", 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7109000000000001e-05 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132474+0000", 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36755078200000002 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132529+0000", 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4880999999999999e-05 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111592 9.2 e155/139 9:410d9e3a:::benchmark_data_smithi042_157727_object111591:head v 155'6952, mlcod=155'6949)", 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.701470839, 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36121327600000003, 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771348+0000", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771349+0000", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.353e-06 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771399+0000", 2024-09-13T14:46:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0086999999999997e-05 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119327+0000", 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34792794599999999 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119345+0000", 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7586000000000001e-05 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.122692+0000", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033467879999999998 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.122713+0000", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0973999999999999e-05 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132537+0000", 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098238549999999994 2024-09-13T14:46:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132561+0000", 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4686999999999999e-05 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111590 9.6 e155/139 9:6fbdf5c6:::benchmark_data_smithi042_157727_object111589:head v 155'6837, mlcod=155'6834)", 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:46:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.70145258900001, 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.35755826299999999, 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771366+0000", 2024-09-13T14:46:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771367+0000", 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.031e-06 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771408+0000", 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0262000000000002e-05 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119343+0000", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34793492199999998 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119354+0000", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1268e-05 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120248+0000", 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00089436300000000002 2024-09-13T14:46:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120269+0000", 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0554999999999999e-05 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128883+0000", 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086147319999999999 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.128925+0000", 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1130000000000001e-05 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111608 9.0 e155/139 9:0bc62cae:::benchmark_data_smithi042_157727_object111607:head v 155'6920, mlcod=155'6918)", 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.683279032, 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34107008799999999, 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789540+0000", 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789541+0000", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.254e-06 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.789594+0000", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2549999999999997e-05 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119370+0000", 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32977673600000001 2024-09-13T14:46:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119381+0000", 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0505e-05 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119981+0000", 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060055600000000003 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.120005+0000", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3292e-05 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.130556+0000", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010551328 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.130610+0000", 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3866999999999998e-05 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111637 9.2 e155/139 9:4caa1eda:::benchmark_data_smithi042_157727_object111636:head v 155'6956, mlcod=155'6949)", 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.35320998899999, 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022970880999999999, 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119609+0000", 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.79e-07 2024-09-13T14:46:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119657+0000", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.795e-05 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119664+0000", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4420000000000003e-06 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119668+0000", 2024-09-13T14:46:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3089999999999997e-06 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.129908+0000", 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010240196 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.129928+0000", 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0316999999999999e-05 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142519+0000", 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012590565 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142580+0000", 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0822999999999997e-05 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:111640 9.2 e155/139 9:44090333:::benchmark_data_smithi042_157727_object111639:head v 155'6957, mlcod=155'6949)", 2024-09-13T14:46:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.35314416099999, 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022948066, 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119675+0000", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119756+0000", 2024-09-13T14:46:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1242000000000006e-05 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119763+0000", 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2929999999999997e-06 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.119767+0000", 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3200000000000001e-06 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132590+0000", 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012823413000000001 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.132612+0000", 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1537000000000001e-05 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142589+0000", 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0099768800000000005 2024-09-13T14:46:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.142623+0000", 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4047000000000001e-05 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:143739 9.0 e158/139 9:0a8a0b03:::benchmark_data_smithi042_157727_object13790:head v 158'8964, mlcod=158'8962)", 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:00.188890+0000", 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 154.283929282, 2024-09-13T14:46:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.037379904999999998, 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188890+0000", 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188890+0000", 2024-09-13T14:46:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188891+0000", 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1260000000000001e-06 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188900+0000", 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4639999999999995e-06 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188919+0000", 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8681999999999998e-05 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188928+0000", 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4669999999999993e-06 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.188992+0000", 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3310000000000002e-05 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.189022+0000", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0318e-05 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226228+0000", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.037206428 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226270+0000", 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1109999999999998e-05 2024-09-13T14:46:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:143743 9.0 e158/139 9:04a8ec00:::benchmark_data_smithi042_157727_object13794:head v 158'8965, mlcod=158'8962)", 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:00.196688+0000", 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 154.27613099999999, 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.029614598999999998, 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196688+0000", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196688+0000", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196689+0000", 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5499999999999997e-07 2024-09-13T14:46:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196695+0000", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5819999999999998e-06 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196710+0000", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4319e-05 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.196717+0000", 2024-09-13T14:46:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2880000000000004e-06 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209947+0000", 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013229925 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209966+0000", 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9151e-05 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226276+0000", 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016309595999999999 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226302+0000", 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6883e-05 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:143753 9.0 e158/139 9:0eca5691:::benchmark_data_smithi042_157727_object13804:head v 158'8966, mlcod=158'8962)", 2024-09-13T14:46:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:00.203409+0000", 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 154.26940986299999, 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022925071000000002, 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203409+0000", 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203409+0000", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203410+0000", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1399999999999996e-07 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203417+0000", 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6730000000000003e-06 2024-09-13T14:46:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203428+0000", 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1212999999999999e-05 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.203439+0000", 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1212999999999999e-05 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.215055+0000", 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011616122 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.215116+0000", 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0683000000000002e-05 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226308+0000", 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011192451000000001 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.226334+0000", 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5902e-05 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:143756 9.0 e158/139 9:0b927860:::benchmark_data_smithi042_157727_object13807:head v 158'8967, mlcod=158'8962)", 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:00.209208+0000", 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 154.263610543, 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.028663271000000001, 2024-09-13T14:46:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209208+0000", 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209208+0000", 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209209+0000", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0449999999999999e-06 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209218+0000", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4330000000000004e-06 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209234+0000", 2024-09-13T14:46:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6521e-05 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.209243+0000", 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6989999999999992e-06 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.222704+0000", 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013460471999999999 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.222756+0000", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2203000000000001e-05 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.237271+0000", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014514954 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:00.237872+0000", 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060094399999999996 2024-09-13T14:46:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:165313 9.2 e162/139 9:4110baf3:::benchmark_data_smithi042_157727_object35364:head v 162'10369, mlcod=162'10364)", 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:22.989958+0000", 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 131.48286038399999, 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.024064882999999999, 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989958+0000", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989958+0000", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989959+0000", 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0800000000000004e-07 2024-09-13T14:46:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989966+0000", 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.579e-06 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989977+0000", 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1415e-05 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.989985+0000", 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1829999999999994e-06 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.990209+0000", 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000223821 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:22.990230+0000", 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1078e-05 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.013900+0000", 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023669778999999998 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.014023+0000", 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012322 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:165866 9.0 e162/139 9:0e5e3689:::benchmark_data_smithi042_157727_object35917:head v 162'10364, mlcod=162'10362)", 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:23.577982+0000", 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 130.894836919, 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023416402999999999, 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.577982+0000", 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.577982+0000", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.577983+0000", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.161e-06 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.577992+0000", 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6989999999999992e-06 2024-09-13T14:46:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.578008+0000", 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6456e-05 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.578016+0000", 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6179999999999997e-06 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.578202+0000", 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018604400000000001 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.578263+0000", 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0600000000000003e-05 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.601355+0000", 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023092464 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:23.601398+0000", 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3361e-05 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:173381 9.6 e164/139 9:633677fe:::benchmark_data_smithi042_157727_object43432:head v 164'10569, mlcod=164'10566)", 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:31.614539+0000", 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 122.858279931, 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022571898, 2024-09-13T14:46:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614539+0000", 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614539+0000", 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614540+0000", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.085e-06 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614548+0000", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.157e-06 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614567+0000", 2024-09-13T14:46:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8802000000000001e-05 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614575+0000", 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4360000000000002e-06 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614653+0000", 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7574e-05 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.614673+0000", 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9911e-05 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.636882+0000", 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022209541999999999 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:31.637111+0000", 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022839099999999999 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.617 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:174608 9.2 e164/139 9:4f4c9ea5:::benchmark_data_smithi042_157727_object44659:head v 164'10892, mlcod=164'10889)", 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:32.960564+0000", 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.512254563, 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.024658472000000001, 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960564+0000", 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960564+0000", 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960566+0000", 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.288e-06 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960574+0000", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9410000000000007e-06 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960590+0000", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6084000000000001e-05 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.960598+0000", 2024-09-13T14:46:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8779999999999998e-06 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.963188+0000", 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025903829999999999 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.963212+0000", 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4530000000000001e-05 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.984462+0000", 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021249292999999999 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.985223+0000", 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00076107499999999995 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.622 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:174625 9.2 e164/139 9:4c27d12d:::benchmark_data_smithi042_157727_object44676:head v 164'10895, mlcod=164'10890)", 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:32.980880+0000", 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.491938938, 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023616049, 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980880+0000", 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980880+0000", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980881+0000", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.181e-06 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980891+0000", 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6050000000000004e-06 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980909+0000", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8770000000000002e-05 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.980923+0000", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3067e-05 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.985665+0000", 2024-09-13T14:46:34.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047421959999999997 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:32.985720+0000", 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5674000000000003e-05 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:33.004346+0000", 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018626039 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:33.004496+0000", 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149517 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:219962 9.2 e172/139 9:41019ae5:::benchmark_data_smithi042_157727_object90013:head v 172'13698, mlcod=172'13694)", 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:20.827036+0000", 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 73.645783304999995, 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.027977264000000002, 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.627 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827036+0000", 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827036+0000", 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827037+0000", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2920000000000001e-06 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827043+0000", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8270000000000003e-06 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827059+0000", 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6429000000000001e-05 2024-09-13T14:46:34.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.827067+0000", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0020000000000006e-06 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.831940+0000", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048731019999999998 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832035+0000", 2024-09-13T14:46:34.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4907000000000005e-05 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.854435+0000", 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022399522000000002 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.855013+0000", 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00057818300000000002 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.631 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:219965 9.2 e172/139 9:49f1b76f:::benchmark_data_smithi042_157727_object90016:head v 172'13699, mlcod=172'13694)", 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:20.832514+0000", 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 73.640304827999998, 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022947564, 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832514+0000", 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832514+0000", 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832515+0000", 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0240000000000001e-06 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832522+0000", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1250000000000004e-06 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832539+0000", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6609e-05 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.832547+0000", 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0469999999999994e-06 2024-09-13T14:46:34.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.842616+0000", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010069470000000001 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.842639+0000", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2809000000000001e-05 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.855017+0000", 2024-09-13T14:46:34.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012378286 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:20.855462+0000", 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00044419400000000002 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:242219 9.6 e173/139 9:62dc7f67:::benchmark_data_smithi042_157727_object112270:head v 173'14829, mlcod=173'14827)", 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:44.227880+0000", 2024-09-13T14:46:34.636 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 50.244938388999998, 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023162261999999999, 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227880+0000", 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227880+0000", 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227881+0000", 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4e-07 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227889+0000", 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7149999999999998e-06 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227906+0000", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6963e-05 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227917+0000", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1066e-05 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.227999+0000", 2024-09-13T14:46:34.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1462e-05 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.228020+0000", 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1098999999999999e-05 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.250938+0000", 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022917963 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:44.251043+0000", 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105154 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.34797.0:247233 9.6 e174/139 9:66de6f81:::benchmark_data_smithi042_157727_object117284:head v 174'15135, mlcod=174'15129)", 2024-09-13T14:46:34.641 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:50.080921+0000", 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 44.391897643, 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023119958999999999, 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080921+0000", 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:34.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080921+0000", 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080922+0000", 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.017e-06 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080933+0000", 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1144999999999999e-05 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080952+0000", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8111999999999999e-05 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.080959+0000", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8350000000000004e-06 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.081171+0000", 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021212 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.081234+0000", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2707999999999997e-05 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.103919+0000", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022684890999999999 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:50.104041+0000", 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012213099999999999 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.645 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.646 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.646 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:34.646 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:34.646 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:46:34.646 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:34.670 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:34.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.797 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:34.898 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:34.923 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:34.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:34.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:35.025 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:35.045 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:35.046 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:35.046 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:35.046 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:35.046 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:46:35.046 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:35.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:35.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:35.053 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:46:35.053 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:35.069 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:46:35.070 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:35.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:35.096 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2051568016' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:35.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:35.096 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2051568016' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:35.099 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:46:35.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:35.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:35.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:35.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:35.400 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.425 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2051568016' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:35.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:35.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:35.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:35.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:35.672 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:35.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:35.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:35.797 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:35.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:35.921 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:35.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:36.023 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:36.049 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:36.150 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:36.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:36.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: pgmap v647: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: pgmap v647: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:36.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:36.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: pgmap v647: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2360104935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: osdmap e183: 8 total, 8 up, 7 in 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:36.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:36.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:36.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:36.657 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:36.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:36.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:36.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:36.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:36.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:36.935 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:36.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:36.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:46:37.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:37.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:37.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:37.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:37.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:37.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:37.315 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:37.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:37.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:37.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:37.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:37.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:37.694 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:37.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:37.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:37.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:37.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:37.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:37.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:38.023 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:38.047 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:38.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:38.172 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:38.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:38.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:38.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:38 smithi042 ceph-mon[108570]: pgmap v649: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:38.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:38 smithi042 ceph-mon[113926]: pgmap v649: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:38.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:38.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:38.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:38 smithi159 ceph-mon[102519]: pgmap v649: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:38.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:38.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:46:38.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:38.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:38.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:38.704 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:38.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:38.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:38.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:38.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:38.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:38.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:39.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.101 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_5' created 2024-09-13T14:46:39.181 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.319 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:46:39.319 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:46:39.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:39.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:39.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:39.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:46:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-13T14:46:39.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:39.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:39.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:39.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:39.831 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:39.933 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:39.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:39.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:39.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:39.964 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:39] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.51.0" 2024-09-13T14:46:39.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-13T14:46:40.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:40.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:40.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[108570]: pgmap v650: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:40.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:46:40.293 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[108570]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:40.293 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[113926]: pgmap v650: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:40.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:46:40.294 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:40 smithi042 ceph-mon[113926]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:40.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:40.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:40.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.459 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:40 smithi159 ceph-mon[102519]: pgmap v650: 83 pgs: 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1875859335' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-13T14:46:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:40 smithi159 ceph-mon[102519]: osdmap e184: 8 total, 8 up, 7 in 2024-09-13T14:46:40.543 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:40.543 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:40.559 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.582 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:40.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:40.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:40.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:40.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:40.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:40.958 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:40.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:41.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:41.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:41.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:41.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:41.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:41.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:41.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:41.332 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:41.340 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:41 smithi042 ceph-mon[113926]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:41.340 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:41 smithi042 ceph-mon[108570]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:41.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:46:41.384 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:41.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:41.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:41.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:41 smithi159 ceph-mon[102519]: osdmap e185: 8 total, 8 up, 7 in 2024-09-13T14:46:41.561 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:41.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:41.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:41.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:41.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:41.812 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:41.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:41.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:41.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:41.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:42.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:42.192 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.216 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:42.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:42.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:42.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:42 smithi159 ceph-mon[102519]: pgmap v653: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:42 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:46:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:42 smithi159 ceph-mon[102519]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:42.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:42.592 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[108570]: pgmap v653: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[108570]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[113926]: pgmap v653: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:46:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:42 smithi042 ceph-mon[113926]: osdmap e186: 8 total, 8 up, 7 in 2024-09-13T14:46:42.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:42.703 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:42.703 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:42.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:42.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:42.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:42.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:42.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:42.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:42.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.102 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:43.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:43.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.225 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:43.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:43.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:43.450 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:43.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:43.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:43.599 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:43.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:43.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:43.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:43.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:43.975 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:43.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:44.077 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:44.103 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:44.120 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:46:44.205 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:44.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:44.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:44.328 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:44.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:44.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:44.479 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:44.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:44 smithi159 ceph-mon[102519]: pgmap v655: 99 pgs: 1 unknown, 98 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/65173153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.518 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[108570]: pgmap v655: 99 pgs: 1 unknown, 98 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:44.519 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/65173153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.519 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.519 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[113926]: pgmap v655: 99 pgs: 1 unknown, 98 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:44.519 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/65173153' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.519 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:46:44.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:44.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:44.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:44.730 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:44.831 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:44.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:44.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:44.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:45.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:45.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:45.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:45.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:45.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:45.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[108570]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:45.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:45 smithi042 ceph-mon[113926]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:45.377 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:45.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:45.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:45.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:45 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:45 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:46:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:45 smithi159 ceph-mon[102519]: osdmap e187: 8 total, 8 up, 7 in 2024-09-13T14:46:45.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:45.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:45.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:45.676 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:45.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:45.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:45.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:45.932 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:45.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:45.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:46.034 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:46.058 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:46.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:46.160 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:46.183 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:46 smithi042 ceph-mon[113926]: pgmap v657: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:46.183 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:46 smithi042 ceph-mon[108570]: pgmap v657: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:46.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:46.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:46.312 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:46.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:46.440 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:46.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:46 smithi159 ceph-mon[102519]: pgmap v657: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:46.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:46.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:46.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:46.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:46.690 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:46.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:46.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:46.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:46.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:46.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:47.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:47.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:46:47.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:47.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:47.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:47.193 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:47.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:47.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:47.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:47.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:47.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:47.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:47.570 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:47.672 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:47.699 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:47.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:47.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:47.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:47.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:47.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:47.950 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:47.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:47.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:48.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:48.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:48.179 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:48.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:48.217 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:46:48.217 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:46:48.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:48.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:48 smithi042 ceph-mon[108570]: pgmap v658: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:48 smithi042 ceph-mon[113926]: pgmap v658: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:48.432 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:48.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:48.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:48 smithi159 ceph-mon[102519]: pgmap v658: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:48.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:48.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:48.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:48.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:48.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:46:48.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:48.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:48.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:48.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:48.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:49.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:49.094 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:49.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:49.137 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-13T14:46:49.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:49.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:49.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:49.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:49.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:49.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:49.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:49.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:46:49.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:49.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:46:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:46:49.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:49.721 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:49.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:49.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:49.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:49.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:49.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:49.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:50.010 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:49] "GET /metrics HTTP/1.1" 200 34035 "" "Prometheus/2.51.0" 2024-09-13T14:46:50.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:50.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:50.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:50.203 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:50.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:50.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[113926]: pgmap v659: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:50.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:46:50.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[113926]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:50.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[108570]: pgmap v659: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:50.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:46:50.265 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:50 smithi042 ceph-mon[108570]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:50.328 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:50.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:50.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:50.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:50.479 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:50.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:50.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:50 smithi159 ceph-mon[102519]: pgmap v659: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4143570386' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:46:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:50 smithi159 ceph-mon[102519]: osdmap e188: 8 total, 8 up, 7 in 2024-09-13T14:46:50.548 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:46:50.549 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:46:50.555 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:46:50.555 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:46:50.555 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:50.576 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:46:50.576 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:46:50.581 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:50.595 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:46:50.595 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:46:50.605 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:50.613 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:46:50.613 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:46:50.629 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:46:50.630 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:46:50.646 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:46:50.647 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:46:50.663 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:46:50.663 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:46:50.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:50.730 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:50.832 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:50.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:50.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:50.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:50.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:50.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:51.037 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-13T14:46:51.037 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-09-13T14:46:51.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:46:51.040 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:46:51.040 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:46:51.085 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:51.092 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:46:51.092 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:46:51.092 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_191918 2024-09-13T14:46:51.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:46:51.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:46:51.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:51.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:51.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:51.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:51.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:51.440 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2772570316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:51.440 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:51.441 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.441 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2772570316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:51.441 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:51.441 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2772570316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:46:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:46:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:46:51.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:51.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:51.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:51.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:51.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:51.821 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:51.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:51.960 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:51.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:51.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:52.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:52.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3905 3889 243.049 243.062 0.00313736 0.00410164 2024-09-13T14:46:52.098 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:52.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:52.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:52.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:52.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:52.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:52.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:52.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:52 smithi159 ceph-mon[102519]: pgmap v661: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:52.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:52 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:46:52.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:52.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:52.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:52 smithi042 ceph-mon[113926]: pgmap v661: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:52.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:52 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:46:52.566 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:52 smithi042 ceph-mon[108570]: pgmap v661: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2024-09-13T14:46:52.566 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:52 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:46:52.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:52.648 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:52.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:52.750 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:52.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:52.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:52.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:52.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:52.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:52.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:53.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.061 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:53.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.066 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:53.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 7610 7594 237.273 231.562 0.00306879 0.00420674 2024-09-13T14:46:53.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.198 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:53.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:53.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:53.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:53.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:53.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:53.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:53.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:53.606 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:53.708 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:53.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:53.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:53.882 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:53.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:53.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:54.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:54.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 11073 11057 230.322 216.438 0.00401032 0.00433534 2024-09-13T14:46:54.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:54.158 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:54.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:54.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:54.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:54.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:54.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:54.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:54 smithi159 ceph-mon[102519]: pgmap v662: 99 pgs: 99 active+clean; 19 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s 2024-09-13T14:46:54.503 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:54 smithi042 ceph-mon[113926]: pgmap v662: 99 pgs: 99 active+clean; 19 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s 2024-09-13T14:46:54.503 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:54 smithi042 ceph-mon[108570]: pgmap v662: 99 pgs: 99 active+clean; 19 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s 2024-09-13T14:46:54.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:54.578 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:54.680 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:54.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:54.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:54.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:54.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:54.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:54.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:54.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:55.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:55.093 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 14259 14243 222.519 199.125 0.00677287 0.00448703 2024-09-13T14:46:55.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:55.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:55.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:55 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:55.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:55 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:55.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:55.407 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:55 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:46:55.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:55.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:55.689 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:55.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:55.728 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:46:55.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:55.832 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:55.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:46:55.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:55.971 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:55.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 15 17236 17221 215.238 186.125 0.00765419 0.00464026 2024-09-13T14:46:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:56.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:56.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[113926]: pgmap v663: 99 pgs: 99 active+clean; 50 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 5.0 MiB/s wr, 79 op/s 2024-09-13T14:46:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[113926]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[108570]: pgmap v663: 99 pgs: 99 active+clean; 50 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 5.0 MiB/s wr, 79 op/s 2024-09-13T14:46:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:56 smithi042 ceph-mon[108570]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:56.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:56.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:56.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:56 smithi159 ceph-mon[102519]: pgmap v663: 99 pgs: 99 active+clean; 50 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 5.0 MiB/s wr, 79 op/s 2024-09-13T14:46:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:56 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:46:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:56 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2794420652' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:46:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:56 smithi159 ceph-mon[102519]: osdmap e189: 8 total, 8 up, 7 in 2024-09-13T14:46:56.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:56.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:56.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:56.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:56.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:56.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:56.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:56.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:56.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:56.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:57.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:46:57.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:46:57.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 20089 20073 209.071 178.25 0.00699187 0.00477785 2024-09-13T14:46:57.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:57.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:57.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:57.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:57.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:57.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:57.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:57.780 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:57.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:57.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:57.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:57.920 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:58.020 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 22789 22773 203.308 168.75 0.00557651 0.00491292 2024-09-13T14:46:58.158 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:58.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:46:58.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:58.454 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:46:58 smithi042 ceph-mon[108570]: pgmap v665: 99 pgs: 99 active+clean; 759 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:46:58.455 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:46:58 smithi042 ceph-mon[113926]: pgmap v665: 99 pgs: 99 active+clean; 759 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:46:58.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:58.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:46:58 smithi159 ceph-mon[102519]: pgmap v665: 99 pgs: 99 active+clean; 759 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:46:58.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:46:58.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:46:58.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:46:58.736 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:46:58.768 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:58.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:46:58.871 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:46:58.873 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:46:58.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:58.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:58.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:84561 9.2 9:41a2a950:::benchmark_data_smithi042_157727_object84560:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:46:58.990 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:01.644434+0000", 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 237.29298231800001, 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.38593548, 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 84561 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:58.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644434+0000", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644434+0000", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644436+0000", 2024-09-13T14:46:58.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9230000000000001e-06 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644459+0000", 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2439999999999999e-05 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644472+0000", 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3685e-05 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644480+0000", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.379e-06 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644538+0000", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8525e-05 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644671+0000", 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013299399999999999 2024-09-13T14:46:58.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.644807+0000", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013611499999999999 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719634+0000", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.074826448000000004 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719660+0000", 2024-09-13T14:46:58.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5599999999999999e-05 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.030217+0000", 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31055751100000001 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.996 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.030320+0000", 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000103329 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.030370+0000", 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9530999999999997e-05 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:84564 9.2 9:489fa143:::benchmark_data_smithi042_157727_object84563:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:01.719904+0000", 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 237.21751273199999, 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31446063600000002, 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 84564 2024-09-13T14:46:58.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719904+0000", 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719904+0000", 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:58.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719906+0000", 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2230000000000001e-06 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719944+0000", 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7876999999999997e-05 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719957+0000", 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3382e-05 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719966+0000", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1389999999999995e-06 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720010+0000", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4747999999999999e-05 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720144+0000", 2024-09-13T14:46:59.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013322099999999999 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720245+0000", 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101499 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.725480+0000", 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052347950000000004 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.725517+0000", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7345000000000002e-05 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.030408+0000", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.30489028699999998 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.030979+0000", 2024-09-13T14:46:59.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00057139099999999998 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.034365+0000", 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033857290000000001 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:84565 9.2 9:44683cb2:::benchmark_data_smithi042_157727_object84564:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:01.719982+0000", 2024-09-13T14:46:59.004 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 237.21743475700001, 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31451728600000001, 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 84565 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.005 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719982+0000", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719982+0000", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.719982+0000", 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1600000000000001e-07 2024-09-13T14:46:59.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720006+0000", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3498000000000001e-05 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720015+0000", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8570000000000001e-06 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.720022+0000", 2024-09-13T14:46:59.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7750000000000004e-06 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.722847+0000", 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028254759999999999 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.722935+0000", 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7639000000000006e-05 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.723055+0000", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120189 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.725587+0000", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025318239999999998 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:01.725853+0000", 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026618900000000001 2024-09-13T14:46:59.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.034411+0000", 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.30855806200000002 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.034461+0000", 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9626000000000002e-05 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:02.034499+0000", 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8634999999999999e-05 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111438 9.2 9:48f1585b:::benchmark_data_smithi042_157727_object111437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.550127+0000", 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.387289544, 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.111118646, 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.011 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111438 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550127+0000", 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550127+0000", 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550128+0000", 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550147+0000", 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9794999999999999e-05 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550154+0000", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3709999999999996e-06 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.550159+0000", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7330000000000003e-06 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.553802+0000", 2024-09-13T14:46:59.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036438579999999998 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.553923+0000", 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120764 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.554081+0000", 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015779300000000001 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.570018+0000", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015937197 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.574891+0000", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048729000000000003 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661133+0000", 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.086241566000000006 2024-09-13T14:46:59.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661190+0000", 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7389000000000002e-05 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661246+0000", 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5812e-05 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111443 9.2 9:496fb4ed:::benchmark_data_smithi042_157727_object111442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.551052+0000", 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.38636466099999, 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.110437598, 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111443 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.018 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551052+0000", 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551052+0000", 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551053+0000", 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7899999999999998e-07 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551075+0000", 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2506000000000001e-05 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551085+0000", 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3640000000000005e-06 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.551091+0000", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7309999999999996e-06 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.560949+0000", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098581380000000007 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.561073+0000", 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012362500000000001 2024-09-13T14:46:59.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.561204+0000", 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013123200000000001 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.574203+0000", 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012998954 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.574900+0000", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00069705899999999998 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661414+0000", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.086513594999999999 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661456+0000", 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1931999999999998e-05 2024-09-13T14:46:59.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.661490+0000", 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3782999999999998e-05 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111577 9.2 9:458b1f83:::benchmark_data_smithi042_157727_object111576:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.758559+0000", 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.178857937, 2024-09-13T14:46:59.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.37886590999999997, 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111577 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758559+0000", 2024-09-13T14:46:59.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758559+0000", 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758560+0000", 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.711e-06 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758582+0000", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1861999999999998e-05 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758594+0000", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2077e-05 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.758602+0000", 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9619999999999993e-06 2024-09-13T14:46:59.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.763900+0000", 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052971839999999999 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764034+0000", 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013403500000000001 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.764625+0000", 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059168399999999998 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.769251+0000", 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046259229999999997 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776118+0000", 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068663559999999997 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137274+0000", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36115679299999998 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137364+0000", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9617999999999995e-05 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137425+0000", 2024-09-13T14:46:59.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0705000000000002e-05 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111592 9.2 9:410d9e3a:::benchmark_data_smithi042_157727_object111591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.770825+0000", 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.16659221200001, 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36675090399999999, 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.031 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111592 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770825+0000", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770825+0000", 2024-09-13T14:46:59.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770825+0000", 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0100000000000005e-07 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770845+0000", 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9877e-05 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770852+0000", 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2049999999999996e-06 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770868+0000", 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6084000000000001e-05 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.770977+0000", 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010919100000000001 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771085+0000", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010747999999999999 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.771216+0000", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013128799999999999 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.775626+0000", 2024-09-13T14:46:59.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004409736 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776341+0000", 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00071528800000000001 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137462+0000", 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36112063900000002 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137510+0000", 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8034000000000001e-05 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137575+0000", 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5580999999999994e-05 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.037 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111600 9.2 9:44cf1321:::benchmark_data_smithi042_157727_object111599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.776899+0000", 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.16051811299999, 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.36084506900000002, 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111600 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776899+0000", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776899+0000", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776900+0000", 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2410000000000001e-06 2024-09-13T14:46:59.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.776938+0000", 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8080999999999997e-05 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.777074+0000", 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013596600000000001 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.777082+0000", 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7319999999999998e-06 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.777127+0000", 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5816999999999999e-05 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.777247+0000", 2024-09-13T14:46:59.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000119405 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.777389+0000", 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014237399999999999 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.781569+0000", 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041797659999999997 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.782469+0000", 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00089987499999999996 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137614+0000", 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.35514535200000003 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137658+0000", 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3473999999999997e-05 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137744+0000", 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5986e-05 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111628 9.2 9:4541aa3f:::benchmark_data_smithi042_157727_object111627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.044 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.805519+0000", 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.13189749700001, 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.33235806699999998, 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111628 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805519+0000", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805519+0000", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805521+0000", 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.438e-06 2024-09-13T14:46:59.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805543+0000", 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2344000000000002e-05 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805559+0000", 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6299000000000001e-05 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805568+0000", 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2619999999999998e-06 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805648+0000", 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0530000000000003e-05 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805759+0000", 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000110788 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.805905+0000", 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014567900000000001 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.809071+0000", 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031666620000000002 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.809792+0000", 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00072121900000000005 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137786+0000", 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32799377499999999 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137832+0000", 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5899000000000002e-05 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137877+0000", 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5172000000000001e-05 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111633 9.2 9:4f832cbb:::benchmark_data_smithi042_157727_object111632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.813243+0000", 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.12417364699999, 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32475416299999998, 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.051 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111633 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813243+0000", 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813243+0000", 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813245+0000", 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7379999999999999e-06 2024-09-13T14:46:59.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813266+0000", 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1035000000000001e-05 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813281+0000", 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5339999999999999e-05 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813288+0000", 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9460000000000001e-06 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813358+0000", 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9809999999999997e-05 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813512+0000", 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153883 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.813692+0000", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017989300000000001 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.817035+0000", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033435100000000001 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818040+0000", 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010049639999999999 2024-09-13T14:46:59.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137917+0000", 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31987673799999999 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137963+0000", 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5695000000000002e-05 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.137997+0000", 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4610999999999998e-05 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111637 9.2 9:4caa1eda:::benchmark_data_smithi042_157727_object111636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.818016+0000", 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.11940118800001, 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32649850499999999, 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.058 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111637 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818016+0000", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818016+0000", 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818017+0000", 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0729999999999999e-06 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818044+0000", 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6956000000000001e-05 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818055+0000", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0951999999999999e-05 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818062+0000", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3989999999999998e-06 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818102+0000", 2024-09-13T14:46:59.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0481000000000001e-05 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818201+0000", 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8121999999999993e-05 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.818295+0000", 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4217999999999998e-05 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.820906+0000", 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026116519999999999 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.821103+0000", 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019668100000000001 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.144400+0000", 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.32329681199999999 2024-09-13T14:46:59.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.144468+0000", 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7674000000000003e-05 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.144514+0000", 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6485e-05 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:111640 9.2 9:44090333:::benchmark_data_smithi042_157727_object111639:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:26.825670+0000", 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.11174695299999, 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31953050700000002, 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.065 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 111640 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825670+0000", 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825670+0000", 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825671+0000", 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.035e-06 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825690+0000", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8921999999999999e-05 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825701+0000", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1042e-05 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825706+0000", 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3279999999999996e-06 2024-09-13T14:46:59.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825757+0000", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1279999999999997e-05 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825835+0000", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7352000000000001e-05 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.825933+0000", 2024-09-13T14:46:59.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8115999999999994e-05 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.828279+0000", 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023461850000000002 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:26.828585+0000", 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00030559100000000002 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.145107+0000", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31652269599999999 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.145161+0000", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3680000000000001e-05 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:27.145200+0000", 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9279999999999997e-05 2024-09-13T14:46:59.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119552 9.3 9:cabe5794:::benchmark_data_smithi042_157727_object119551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 204.02502674999999, 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0227897889999999, 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119552 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912392+0000", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.812e-06 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.915992+0000", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036003739999999999 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.916006+0000", 2024-09-13T14:46:59.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3910000000000001e-05 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.916015+0000", 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4889999999999995e-06 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.922079+0000", 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006064687 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:59.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.922090+0000", 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0404000000000001e-05 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.914897+0000", 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99280757100000006 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.917354+0000", 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002456429 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.917594+0000", 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023996900000000001 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.934990+0000", 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017396838000000001 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935003+0000", 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2974e-05 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935089+0000", 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5782e-05 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935139+0000", 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9523e-05 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935180+0000", 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1026999999999999e-05 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119593 9.3 9:cfa3dde9:::benchmark_data_smithi042_157727_object119592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 203.974524583, 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97686230799999996, 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119593 2024-09-13T14:46:59.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962894+0000", 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.731e-06 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962919+0000", 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4768000000000001e-05 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962937+0000", 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8793000000000001e-05 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962945+0000", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6210000000000004e-06 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.963030+0000", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4834999999999997e-05 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.963040+0000", 2024-09-13T14:46:59.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0118e-05 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.927762+0000", 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96472230299999995 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.927904+0000", 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014198300000000001 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.928063+0000", 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015882299999999999 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935052+0000", 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069884939999999996 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935225+0000", 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017373900000000001 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939647+0000", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044221110000000003 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939703+0000", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5618000000000002e-05 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939754+0000", 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1371e-05 2024-09-13T14:46:59.087 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119630 9.3 9:c9739907:::benchmark_data_smithi042_157727_object119629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:35.010429+0000", 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 203.926987242, 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.93573302300000005, 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.088 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119630 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010429+0000", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010429+0000", 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010431+0000", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.333e-06 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010490+0000", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9497000000000003e-05 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010504+0000", 2024-09-13T14:46:59.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3434e-05 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010511+0000", 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5580000000000003e-06 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010611+0000", 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010016700000000001 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.010622+0000", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0832999999999999e-05 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.931039+0000", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92041687000000005 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.931246+0000", 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000206842 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.931451+0000", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020522499999999999 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935259+0000", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003808241 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.943633+0000", 2024-09-13T14:46:59.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0083737629999999993 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.945993+0000", 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023596609999999999 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.946107+0000", 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011445 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.946162+0000", 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5148999999999997e-05 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:198022 9.3 9:caba68ce:::benchmark_data_smithi042_157727_object68073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:57.728798+0000", 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.208618538, 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19152754, 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.095 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 198022 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728798+0000", 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728798+0000", 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728800+0000", 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5489999999999999e-06 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728806+0000", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1009999999999997e-06 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728884+0000", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8221999999999997e-05 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.728894+0000", 2024-09-13T14:46:59.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9059999999999997e-06 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.730614+0000", 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0017200539999999999 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.730721+0000", 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000106977 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.730827+0000", 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010600800000000001 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.734587+0000", 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037600670000000002 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.737628+0000", 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030410810000000002 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.920236+0000", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18260817300000001 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.920282+0000", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5982999999999997e-05 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.920326+0000", 2024-09-13T14:46:59.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3418999999999999e-05 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:198031 9.3 9:c18d4e6b:::benchmark_data_smithi042_157727_object68082:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:57.738722+0000", 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.19869510300001, 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.188779631, 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.102 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 198031 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738722+0000", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738722+0000", 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738723+0000", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0899999999999999e-06 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738728+0000", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8459999999999999e-06 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738740+0000", 2024-09-13T14:46:59.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2258e-05 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738745+0000", 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5269999999999998e-06 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738796+0000", 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.079e-05 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.738948+0000", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000151543 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.739096+0000", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000148329 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741786+0000", 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002689964 2024-09-13T14:46:59.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.742251+0000", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00046539699999999999 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927400+0000", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.185148543 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927456+0000", 2024-09-13T14:46:59.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5726000000000001e-05 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927501+0000", 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5618000000000003e-05 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:198034 9.2 9:418c0d1b:::benchmark_data_smithi042_157727_object68085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:57.741803+0000", 2024-09-13T14:46:59.108 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.195613662, 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18909514399999999, 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 198034 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741803+0000", 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741803+0000", 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741804+0000", 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.339e-06 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741809+0000", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9030000000000003e-06 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741832+0000", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3048e-05 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741840+0000", 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3019999999999997e-06 2024-09-13T14:46:59.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.741903+0000", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3047999999999997e-05 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.742012+0000", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109701 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.742133+0000", 2024-09-13T14:46:59.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120566 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.745144+0000", 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030106439999999998 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.745455+0000", 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031151099999999999 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.930804+0000", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.185348455 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.930851+0000", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7336000000000002e-05 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.930898+0000", 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7290999999999998e-05 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.114 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:198038 9.3 9:c41d3e50:::benchmark_data_smithi042_157727_object68089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:44:57.752209+0000", 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 121.18520774700001, 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17539992500000001, 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.115 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 198038 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752209+0000", 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752209+0000", 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752210+0000", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3200000000000003e-07 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752214+0000", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9570000000000004e-06 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752223+0000", 2024-09-13T14:46:59.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5240000000000006e-06 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752228+0000", 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9490000000000004e-06 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752265+0000", 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6477999999999999e-05 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752348+0000", 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2995000000000001e-05 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.752436+0000", 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7855999999999994e-05 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.755244+0000", 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028086560000000001 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.755251+0000", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6460000000000004e-06 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927536+0000", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17228512300000001 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927578+0000", 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1627e-05 2024-09-13T14:46:59.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:44:57.927609+0000", 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1182000000000001e-05 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:245431 9.3 9:c9d55333:::benchmark_data_smithi042_157727_object115482:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e173)", 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:45:47.936064+0000", 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 71.001353140999996, 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13142411700000001, 2024-09-13T14:46:59.121 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 245431 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936064+0000", 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936064+0000", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936065+0000", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4190000000000001e-06 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936069+0000", 2024-09-13T14:46:59.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8990000000000003e-06 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936081+0000", 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1970999999999999e-05 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936088+0000", 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9120000000000001e-06 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936292+0000", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000204272 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936397+0000", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000104765 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:47.936502+0000", 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105301 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:48.013702+0000", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.077200191000000001 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:48.016655+0000", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00295247 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:48.062918+0000", 2024-09-13T14:46:59.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.046263526999999999 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:48.063139+0000", 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022048099999999999 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:45:48.067488+0000", 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043489089999999998 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.127 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:46:59.128 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:46:59.128 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:46:59.128 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 25315 25299 197.627 157.875 0.0063995 0.00505441 2024-09-13T14:46:59.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:59.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:46:59.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:59.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:59.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:59.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:59.540 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:46:59.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:46:59.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:46:59.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:59.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:46:59.759 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:46:59.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:46:59.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:46:59.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:46:59.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:46:59.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:46:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:46:59.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:00.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:00.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 27670 27654 192.021 147.188 0.00231259 0.00516806 2024-09-13T14:47:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:46:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:46:59] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.51.0" 2024-09-13T14:47:00.114 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:00.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:00.216 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:00.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:00.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:00 smithi159 ceph-mon[102519]: pgmap v666: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:47:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:00 smithi159 ceph-mon[102519]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:00.517 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:00 smithi042 ceph-mon[108570]: pgmap v666: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:47:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:00 smithi042 ceph-mon[108570]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:00 smithi042 ceph-mon[113926]: pgmap v666: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-13T14:47:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:00 smithi042 ceph-mon[113926]: from='client.45044 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:47:00.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:00.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:00.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:00.793 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:00.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:00.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:00.931 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:00.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:00.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:01.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:01.069 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:01.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 29914 29898 186.843 140.25 0.00368221 0.00534639 2024-09-13T14:47:01.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:01.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:01.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:01.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:01.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:01.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:01.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:01.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:01.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:01.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:01.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:01.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:01.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:01.993 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:02.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:02.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 32141 32125 182.51 139.188 0.00657983 0.00547362 2024-09-13T14:47:02.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:02.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:02.301 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.303 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:02.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:02.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:02.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:02.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:02 smithi159 ceph-mon[102519]: pgmap v667: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:47:02.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:02.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:02 smithi042 ceph-mon[113926]: pgmap v667: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:47:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:02 smithi042 ceph-mon[108570]: pgmap v667: 99 pgs: 99 active+clean; 1.1 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T14:47:02.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:02.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:02.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:02.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:02.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:02.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:02.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:03.093 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:03.094 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 34313 34297 178.612 135.75 0.0051488 0.00559338 2024-09-13T14:47:03.131 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:03.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:47:03.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:03.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:03.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:03.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:03.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.517 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:03.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:03.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:03.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:03.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:03.840 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:03.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:03.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:03.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:03.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:04.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:04.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 36206 36190 173.967 118.312 0.0063277 0.00574185 2024-09-13T14:47:04.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:04.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:04.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:04.223 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":666,"stamp":"2024-09-13T14:46:58.281303+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1132402661,"num_objects":17296,"num_object_clones":0,"num_object_copies":51888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17334,"num_write_kb":1105313,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":694,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18968,"ondisk_log_size":18968,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6789352,"kb_used_data":3319136,"kb_used_omap":22,"kb_used_meta":3470121,"kb_avail":649426712,"statfs":{"total":671965249536,"available":665012953088,"internally_reserved":0,"allocated":3398795264,"data_stored":3392330849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":3553404135},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,7,6,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1131806720,"num_objects":17270,"num_object_clones":0,"num_object_copies":51810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17270,"num_write_kb":1105280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.144116"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168954+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:46:55.168954+0000","last_peered":"2024-09-13T14:46:55.168954+0000","last_clean":"2024-09-13T14:46:55.168954+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:46:55.168954+0000","last_undegraded":"2024-09-13T14:46:55.168954+0000","last_fullsized":"2024-09-13T14:46:55.168954+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":174,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.169086+0000","last_change":"2024-09-13T14:43:35.907052+0000","last_active":"2024-09-13T14:46:55.169086+0000","last_peered":"2024-09-13T14:46:55.169086+0000","last_clean":"2024-09-13T14:46:55.169086+0000","last_became_active":"2024-09-13T14:43:35.906588+0000","last_became_peered":"2024-09-13T14:43:35.906588+0000","last_unstale":"2024-09-13T14:46:55.169086+0000","last_undegraded":"2024-09-13T14:46:55.169086+0000","last_fullsized":"2024-09-13T14:46:55.169086+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:56.521107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":153,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288931+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:46:49.288931+0000","last_peered":"2024-09-13T14:46:49.288931+0000","last_clean":"2024-09-13T14:46:49.288931+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:46:49.288931+0000","last_undegraded":"2024-09-13T14:46:49.288931+0000","last_fullsized":"2024-09-13T14:46:49.288931+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167245+0000","last_change":"2024-09-13T14:46:13.037152+0000","last_active":"2024-09-13T14:46:55.167245+0000","last_peered":"2024-09-13T14:46:55.167245+0000","last_clean":"2024-09-13T14:46:55.167245+0000","last_became_active":"2024-09-13T14:46:13.036098+0000","last_became_peered":"2024-09-13T14:46:13.036098+0000","last_unstale":"2024-09-13T14:46:55.167245+0000","last_undegraded":"2024-09-13T14:46:55.167245+0000","last_fullsized":"2024-09-13T14:46:55.167245+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:40.662916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":142,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168499+0000","last_change":"2024-09-13T14:46:13.036628+0000","last_active":"2024-09-13T14:46:55.168499+0000","last_peered":"2024-09-13T14:46:55.168499+0000","last_clean":"2024-09-13T14:46:55.168499+0000","last_became_active":"2024-09-13T14:46:13.035833+0000","last_became_peered":"2024-09-13T14:46:13.035833+0000","last_unstale":"2024-09-13T14:46:55.168499+0000","last_undegraded":"2024-09-13T14:46:55.168499+0000","last_fullsized":"2024-09-13T14:46:55.168499+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:12:02.205425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289648+0000","last_change":"2024-09-13T14:43:35.907589+0000","last_active":"2024-09-13T14:46:49.289648+0000","last_peered":"2024-09-13T14:46:49.289648+0000","last_clean":"2024-09-13T14:46:49.289648+0000","last_became_active":"2024-09-13T14:43:35.907254+0000","last_became_peered":"2024-09-13T14:43:35.907254+0000","last_unstale":"2024-09-13T14:46:49.289648+0000","last_undegraded":"2024-09-13T14:46:49.289648+0000","last_fullsized":"2024-09-13T14:46:49.289648+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:13.369649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":224,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168796+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:46:55.168796+0000","last_peered":"2024-09-13T14:46:55.168796+0000","last_clean":"2024-09-13T14:46:55.168796+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:46:55.168796+0000","last_undegraded":"2024-09-13T14:46:55.168796+0000","last_fullsized":"2024-09-13T14:46:55.168796+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":183,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289977+0000","last_change":"2024-09-13T14:46:13.047307+0000","last_active":"2024-09-13T14:46:49.289977+0000","last_peered":"2024-09-13T14:46:49.289977+0000","last_clean":"2024-09-13T14:46:49.289977+0000","last_became_active":"2024-09-13T14:46:13.033970+0000","last_became_peered":"2024-09-13T14:46:13.033970+0000","last_unstale":"2024-09-13T14:46:49.289977+0000","last_undegraded":"2024-09-13T14:46:49.289977+0000","last_fullsized":"2024-09-13T14:46:49.289977+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:58:39.941061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"189'1070","reported_seq":1084,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.189331+0000","last_change":"2024-09-13T14:46:40.112760+0000","last_active":"2024-09-13T14:46:56.189331+0000","last_peered":"2024-09-13T14:46:56.189331+0000","last_clean":"2024-09-13T14:46:56.189331+0000","last_became_active":"2024-09-13T14:46:40.112223+0000","last_became_peered":"2024-09-13T14:46:40.112223+0000","last_unstale":"2024-09-13T14:46:56.189331+0000","last_undegraded":"2024-09-13T14:46:56.189331+0000","last_fullsized":"2024-09-13T14:46:56.189331+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1070,"log_dups_size":0,"ondisk_log_size":1070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:46:31.840349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70123520,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1070,"num_write_kb":68480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":70,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288147+0000","last_change":"2024-09-13T14:46:13.040552+0000","last_active":"2024-09-13T14:46:49.288147+0000","last_peered":"2024-09-13T14:46:49.288147+0000","last_clean":"2024-09-13T14:46:49.288147+0000","last_became_active":"2024-09-13T14:46:13.033184+0000","last_became_peered":"2024-09-13T14:46:13.033184+0000","last_unstale":"2024-09-13T14:46:49.288147+0000","last_undegraded":"2024-09-13T14:46:49.288147+0000","last_fullsized":"2024-09-13T14:46:49.288147+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:21:53.837595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":140,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288870+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:46:49.288870+0000","last_peered":"2024-09-13T14:46:49.288870+0000","last_clean":"2024-09-13T14:46:49.288870+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:46:49.288870+0000","last_undegraded":"2024-09-13T14:46:49.288870+0000","last_fullsized":"2024-09-13T14:46:49.288870+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"189'1002","reported_seq":1016,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.916725+0000","last_change":"2024-09-13T14:46:40.112220+0000","last_active":"2024-09-13T14:46:55.916725+0000","last_peered":"2024-09-13T14:46:55.916725+0000","last_clean":"2024-09-13T14:46:55.916725+0000","last_became_active":"2024-09-13T14:46:40.111924+0000","last_became_peered":"2024-09-13T14:46:40.111924+0000","last_unstale":"2024-09-13T14:46:55.916725+0000","last_undegraded":"2024-09-13T14:46:55.916725+0000","last_fullsized":"2024-09-13T14:46:55.916725+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1002,"log_dups_size":0,"ondisk_log_size":1002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:26.618016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1002,"num_write_kb":64128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":57,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289729+0000","last_change":"2024-09-13T14:43:35.905294+0000","last_active":"2024-09-13T14:46:49.289729+0000","last_peered":"2024-09-13T14:46:49.289729+0000","last_clean":"2024-09-13T14:46:49.289729+0000","last_became_active":"2024-09-13T14:43:35.904851+0000","last_became_peered":"2024-09-13T14:43:35.904851+0000","last_unstale":"2024-09-13T14:46:49.289729+0000","last_undegraded":"2024-09-13T14:46:49.289729+0000","last_fullsized":"2024-09-13T14:46:49.289729+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:21.591225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.358897+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:46:51.358897+0000","last_peered":"2024-09-13T14:46:51.358897+0000","last_clean":"2024-09-13T14:46:51.358897+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:46:51.358897+0000","last_undegraded":"2024-09-13T14:46:51.358897+0000","last_fullsized":"2024-09-13T14:46:51.358897+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"189'1104","reported_seq":1118,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.356703+0000","last_change":"2024-09-13T14:46:40.114090+0000","last_active":"2024-09-13T14:46:56.356703+0000","last_peered":"2024-09-13T14:46:56.356703+0000","last_clean":"2024-09-13T14:46:56.356703+0000","last_became_active":"2024-09-13T14:46:40.113738+0000","last_became_peered":"2024-09-13T14:46:40.113738+0000","last_unstale":"2024-09-13T14:46:56.356703+0000","last_undegraded":"2024-09-13T14:46:56.356703+0000","last_fullsized":"2024-09-13T14:46:56.356703+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1104,"log_dups_size":0,"ondisk_log_size":1104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:23:00.443194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351744,"num_objects":1104,"num_object_clones":0,"num_object_copies":3312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1104,"num_write_kb":70656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":57,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288873+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:46:49.288873+0000","last_peered":"2024-09-13T14:46:49.288873+0000","last_clean":"2024-09-13T14:46:49.288873+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:46:49.288873+0000","last_undegraded":"2024-09-13T14:46:49.288873+0000","last_fullsized":"2024-09-13T14:46:49.288873+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.169612+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:46:55.169612+0000","last_peered":"2024-09-13T14:46:55.169612+0000","last_clean":"2024-09-13T14:46:55.169612+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:46:55.169612+0000","last_undegraded":"2024-09-13T14:46:55.169612+0000","last_fullsized":"2024-09-13T14:46:55.169612+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"189'985","reported_seq":1006,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.477912+0000","last_change":"2024-09-13T14:46:40.217829+0000","last_active":"2024-09-13T14:46:55.477912+0000","last_peered":"2024-09-13T14:46:55.477912+0000","last_clean":"2024-09-13T14:46:55.477912+0000","last_became_active":"2024-09-13T14:46:40.217475+0000","last_became_peered":"2024-09-13T14:46:40.217475+0000","last_unstale":"2024-09-13T14:46:55.477912+0000","last_undegraded":"2024-09-13T14:46:55.477912+0000","last_fullsized":"2024-09-13T14:46:55.477912+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":985,"log_dups_size":0,"ondisk_log_size":985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:50.448602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64552960,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":985,"num_write_kb":63040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288831+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:46:49.288831+0000","last_peered":"2024-09-13T14:46:49.288831+0000","last_clean":"2024-09-13T14:46:49.288831+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:46:49.288831+0000","last_undegraded":"2024-09-13T14:46:49.288831+0000","last_fullsized":"2024-09-13T14:46:49.288831+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":167,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167977+0000","last_change":"2024-09-13T14:46:13.033214+0000","last_active":"2024-09-13T14:46:55.167977+0000","last_peered":"2024-09-13T14:46:55.167977+0000","last_clean":"2024-09-13T14:46:55.167977+0000","last_became_active":"2024-09-13T14:46:13.032747+0000","last_became_peered":"2024-09-13T14:46:13.032747+0000","last_unstale":"2024-09-13T14:46:55.167977+0000","last_undegraded":"2024-09-13T14:46:55.167977+0000","last_fullsized":"2024-09-13T14:46:55.167977+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:57.749176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"189'1437","reported_seq":1458,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:58.138990+0000","last_change":"2024-09-13T14:46:40.111420+0000","last_active":"2024-09-13T14:46:58.138990+0000","last_peered":"2024-09-13T14:46:58.138990+0000","last_clean":"2024-09-13T14:46:58.138990+0000","last_became_active":"2024-09-13T14:46:40.111121+0000","last_became_peered":"2024-09-13T14:46:40.111121+0000","last_unstale":"2024-09-13T14:46:58.138990+0000","last_undegraded":"2024-09-13T14:46:58.138990+0000","last_fullsized":"2024-09-13T14:46:58.138990+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1437,"log_dups_size":0,"ondisk_log_size":1437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:51:29.119337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94175232,"num_objects":1437,"num_object_clones":0,"num_object_copies":4311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1437,"num_write_kb":91968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":548,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166260+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:46:55.166260+0000","last_peered":"2024-09-13T14:46:55.166260+0000","last_clean":"2024-09-13T14:46:55.166260+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:46:55.166260+0000","last_undegraded":"2024-09-13T14:46:55.166260+0000","last_fullsized":"2024-09-13T14:46:55.166260+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":197,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167717+0000","last_change":"2024-09-13T14:46:13.046297+0000","last_active":"2024-09-13T14:46:55.167717+0000","last_peered":"2024-09-13T14:46:55.167717+0000","last_clean":"2024-09-13T14:46:55.167717+0000","last_became_active":"2024-09-13T14:46:13.037338+0000","last_became_peered":"2024-09-13T14:46:13.037338+0000","last_unstale":"2024-09-13T14:46:55.167717+0000","last_undegraded":"2024-09-13T14:46:55.167717+0000","last_fullsized":"2024-09-13T14:46:55.167717+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:38.124256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.290208+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:46:49.290208+0000","last_peered":"2024-09-13T14:46:49.290208+0000","last_clean":"2024-09-13T14:46:49.290208+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:46:49.290208+0000","last_undegraded":"2024-09-13T14:46:49.290208+0000","last_fullsized":"2024-09-13T14:46:49.290208+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"189'935","reported_seq":956,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.475138+0000","last_change":"2024-09-13T14:46:40.109232+0000","last_active":"2024-09-13T14:46:55.475138+0000","last_peered":"2024-09-13T14:46:55.475138+0000","last_clean":"2024-09-13T14:46:55.475138+0000","last_became_active":"2024-09-13T14:46:40.108968+0000","last_became_peered":"2024-09-13T14:46:40.108968+0000","last_unstale":"2024-09-13T14:46:55.475138+0000","last_undegraded":"2024-09-13T14:46:55.475138+0000","last_fullsized":"2024-09-13T14:46:55.475138+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":935,"log_dups_size":0,"ondisk_log_size":935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:46:51.127103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":935,"num_write_kb":59840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":424,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288209+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:46:49.288209+0000","last_peered":"2024-09-13T14:46:49.288209+0000","last_clean":"2024-09-13T14:46:49.288209+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:46:49.288209+0000","last_undegraded":"2024-09-13T14:46:49.288209+0000","last_fullsized":"2024-09-13T14:46:49.288209+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":177,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.169653+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:46:55.169653+0000","last_peered":"2024-09-13T14:46:55.169653+0000","last_clean":"2024-09-13T14:46:55.169653+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:46:55.169653+0000","last_undegraded":"2024-09-13T14:46:55.169653+0000","last_fullsized":"2024-09-13T14:46:55.169653+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289239+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:46:49.289239+0000","last_peered":"2024-09-13T14:46:49.289239+0000","last_clean":"2024-09-13T14:46:49.289239+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:46:49.289239+0000","last_undegraded":"2024-09-13T14:46:49.289239+0000","last_fullsized":"2024-09-13T14:46:49.289239+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"189'1062","reported_seq":1074,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.910301+0000","last_change":"2024-09-13T14:46:40.114392+0000","last_active":"2024-09-13T14:46:55.910301+0000","last_peered":"2024-09-13T14:46:55.910301+0000","last_clean":"2024-09-13T14:46:55.910301+0000","last_became_active":"2024-09-13T14:46:40.113815+0000","last_became_peered":"2024-09-13T14:46:40.113815+0000","last_unstale":"2024-09-13T14:46:55.910301+0000","last_undegraded":"2024-09-13T14:46:55.910301+0000","last_fullsized":"2024-09-13T14:46:55.910301+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1062,"log_dups_size":0,"ondisk_log_size":1062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:41:05.731116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69599232,"num_objects":1062,"num_object_clones":0,"num_object_copies":3186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1062,"num_write_kb":67968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":157,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:51.358914+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:46:51.358914+0000","last_peered":"2024-09-13T14:46:51.358914+0000","last_clean":"2024-09-13T14:46:51.358914+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:46:51.358914+0000","last_undegraded":"2024-09-13T14:46:51.358914+0000","last_fullsized":"2024-09-13T14:46:51.358914+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289792+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:46:49.289792+0000","last_peered":"2024-09-13T14:46:49.289792+0000","last_clean":"2024-09-13T14:46:49.289792+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:46:49.289792+0000","last_undegraded":"2024-09-13T14:46:49.289792+0000","last_fullsized":"2024-09-13T14:46:49.289792+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"189'1105","reported_seq":1118,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.359590+0000","last_change":"2024-09-13T14:46:40.219022+0000","last_active":"2024-09-13T14:46:56.359590+0000","last_peered":"2024-09-13T14:46:56.359590+0000","last_clean":"2024-09-13T14:46:56.359590+0000","last_became_active":"2024-09-13T14:46:40.218732+0000","last_became_peered":"2024-09-13T14:46:40.218732+0000","last_unstale":"2024-09-13T14:46:56.359590+0000","last_undegraded":"2024-09-13T14:46:56.359590+0000","last_fullsized":"2024-09-13T14:46:56.359590+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1105,"log_dups_size":0,"ondisk_log_size":1105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:10.645449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72417280,"num_objects":1105,"num_object_clones":0,"num_object_copies":3315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1105,"num_write_kb":70720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":424,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288603+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:46:49.288603+0000","last_peered":"2024-09-13T14:46:49.288603+0000","last_clean":"2024-09-13T14:46:49.288603+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:46:49.288603+0000","last_undegraded":"2024-09-13T14:46:49.288603+0000","last_fullsized":"2024-09-13T14:46:49.288603+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166942+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:46:55.166942+0000","last_peered":"2024-09-13T14:46:55.166942+0000","last_clean":"2024-09-13T14:46:55.166942+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:46:55.166942+0000","last_undegraded":"2024-09-13T14:46:55.166942+0000","last_fullsized":"2024-09-13T14:46:55.166942+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168064+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:46:55.168064+0000","last_peered":"2024-09-13T14:46:55.168064+0000","last_clean":"2024-09-13T14:46:55.168064+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:46:55.168064+0000","last_undegraded":"2024-09-13T14:46:55.168064+0000","last_fullsized":"2024-09-13T14:46:55.168064+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"189'1043","reported_seq":1057,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.915418+0000","last_change":"2024-09-13T14:46:40.114484+0000","last_active":"2024-09-13T14:46:55.915418+0000","last_peered":"2024-09-13T14:46:55.915418+0000","last_clean":"2024-09-13T14:46:55.915418+0000","last_became_active":"2024-09-13T14:46:40.114038+0000","last_became_peered":"2024-09-13T14:46:40.114038+0000","last_unstale":"2024-09-13T14:46:55.915418+0000","last_undegraded":"2024-09-13T14:46:55.915418+0000","last_fullsized":"2024-09-13T14:46:55.915418+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1043,"log_dups_size":0,"ondisk_log_size":1043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:32:32.279864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68354048,"num_objects":1043,"num_object_clones":0,"num_object_copies":3129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1043,"num_write_kb":66752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":403,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.287884+0000","last_change":"2024-09-13T14:46:23.741730+0000","last_active":"2024-09-13T14:46:49.287884+0000","last_peered":"2024-09-13T14:46:49.287884+0000","last_clean":"2024-09-13T14:46:49.287884+0000","last_became_active":"2024-09-13T14:46:16.047188+0000","last_became_peered":"2024-09-13T14:46:16.047188+0000","last_unstale":"2024-09-13T14:46:49.287884+0000","last_undegraded":"2024-09-13T14:46:49.287884+0000","last_fullsized":"2024-09-13T14:46:49.287884+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:57:00.809081+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":170,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288461+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:46:49.288461+0000","last_peered":"2024-09-13T14:46:49.288461+0000","last_clean":"2024-09-13T14:46:49.288461+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:46:49.288461+0000","last_undegraded":"2024-09-13T14:46:49.288461+0000","last_fullsized":"2024-09-13T14:46:49.288461+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":186,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167105+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:46:55.167105+0000","last_peered":"2024-09-13T14:46:55.167105+0000","last_clean":"2024-09-13T14:46:55.167105+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:46:55.167105+0000","last_undegraded":"2024-09-13T14:46:55.167105+0000","last_fullsized":"2024-09-13T14:46:55.167105+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289016+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:46:49.289016+0000","last_peered":"2024-09-13T14:46:49.289016+0000","last_clean":"2024-09-13T14:46:49.289016+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:46:49.289016+0000","last_undegraded":"2024-09-13T14:46:49.289016+0000","last_fullsized":"2024-09-13T14:46:49.289016+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"189'970","reported_seq":991,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.473162+0000","last_change":"2024-09-13T14:46:40.217942+0000","last_active":"2024-09-13T14:46:55.473162+0000","last_peered":"2024-09-13T14:46:55.473162+0000","last_clean":"2024-09-13T14:46:55.473162+0000","last_became_active":"2024-09-13T14:46:40.217598+0000","last_became_peered":"2024-09-13T14:46:40.217598+0000","last_unstale":"2024-09-13T14:46:55.473162+0000","last_undegraded":"2024-09-13T14:46:55.473162+0000","last_fullsized":"2024-09-13T14:46:55.473162+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":970,"log_dups_size":0,"ondisk_log_size":970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:10:51.117722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63569920,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":970,"num_write_kb":62080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":458,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167854+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:46:55.167854+0000","last_peered":"2024-09-13T14:46:55.167854+0000","last_clean":"2024-09-13T14:46:55.167854+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:46:55.167854+0000","last_undegraded":"2024-09-13T14:46:55.167854+0000","last_fullsized":"2024-09-13T14:46:55.167854+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":178,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167292+0000","last_change":"2024-09-13T14:46:13.034471+0000","last_active":"2024-09-13T14:46:55.167292+0000","last_peered":"2024-09-13T14:46:55.167292+0000","last_clean":"2024-09-13T14:46:55.167292+0000","last_became_active":"2024-09-13T14:46:13.034047+0000","last_became_peered":"2024-09-13T14:46:13.034047+0000","last_unstale":"2024-09-13T14:46:55.167292+0000","last_undegraded":"2024-09-13T14:46:55.167292+0000","last_fullsized":"2024-09-13T14:46:55.167292+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:13:43.756936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289563+0000","last_change":"2024-09-13T14:46:13.029929+0000","last_active":"2024-09-13T14:46:49.289563+0000","last_peered":"2024-09-13T14:46:49.289563+0000","last_clean":"2024-09-13T14:46:49.289563+0000","last_became_active":"2024-09-13T14:46:13.029600+0000","last_became_peered":"2024-09-13T14:46:13.029600+0000","last_unstale":"2024-09-13T14:46:49.289563+0000","last_undegraded":"2024-09-13T14:46:49.289563+0000","last_fullsized":"2024-09-13T14:46:49.289563+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:23.660781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167711+0000","last_change":"2024-09-13T14:46:13.038783+0000","last_active":"2024-09-13T14:46:55.167711+0000","last_peered":"2024-09-13T14:46:55.167711+0000","last_clean":"2024-09-13T14:46:55.167711+0000","last_became_active":"2024-09-13T14:46:13.038159+0000","last_became_peered":"2024-09-13T14:46:13.038159+0000","last_unstale":"2024-09-13T14:46:55.167711+0000","last_undegraded":"2024-09-13T14:46:55.167711+0000","last_fullsized":"2024-09-13T14:46:55.167711+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:56:07.625807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"189'1128","reported_seq":1141,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.189715+0000","last_change":"2024-09-13T14:46:40.218541+0000","last_active":"2024-09-13T14:46:56.189715+0000","last_peered":"2024-09-13T14:46:56.189715+0000","last_clean":"2024-09-13T14:46:56.189715+0000","last_became_active":"2024-09-13T14:46:40.218131+0000","last_became_peered":"2024-09-13T14:46:40.218131+0000","last_unstale":"2024-09-13T14:46:56.189715+0000","last_undegraded":"2024-09-13T14:46:56.189715+0000","last_fullsized":"2024-09-13T14:46:56.189715+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1128,"log_dups_size":0,"ondisk_log_size":1128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:43:21.630823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73924608,"num_objects":1128,"num_object_clones":0,"num_object_copies":3384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1128,"num_write_kb":72192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":502,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289781+0000","last_change":"2024-09-13T14:46:25.612749+0000","last_active":"2024-09-13T14:46:49.289781+0000","last_peered":"2024-09-13T14:46:49.289781+0000","last_clean":"2024-09-13T14:46:49.289781+0000","last_became_active":"2024-09-13T14:46:16.046554+0000","last_became_peered":"2024-09-13T14:46:16.046554+0000","last_unstale":"2024-09-13T14:46:49.289781+0000","last_undegraded":"2024-09-13T14:46:49.289781+0000","last_fullsized":"2024-09-13T14:46:49.289781+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:25.612649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:25.612649+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:35:13.270953+0000","scrub_duration":0.001601693,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":230,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.171310+0000","last_change":"2024-09-13T14:46:13.025772+0000","last_active":"2024-09-13T14:46:55.171310+0000","last_peered":"2024-09-13T14:46:55.171310+0000","last_clean":"2024-09-13T14:46:55.171310+0000","last_became_active":"2024-09-13T14:46:13.025418+0000","last_became_peered":"2024-09-13T14:46:13.025418+0000","last_unstale":"2024-09-13T14:46:55.171310+0000","last_undegraded":"2024-09-13T14:46:55.171310+0000","last_fullsized":"2024-09-13T14:46:55.171310+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:34.204897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":197,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167556+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:46:55.167556+0000","last_peered":"2024-09-13T14:46:55.167556+0000","last_clean":"2024-09-13T14:46:55.167556+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:46:55.167556+0000","last_undegraded":"2024-09-13T14:46:55.167556+0000","last_fullsized":"2024-09-13T14:46:55.167556+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":241,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167868+0000","last_change":"2024-09-13T14:46:13.033037+0000","last_active":"2024-09-13T14:46:55.167868+0000","last_peered":"2024-09-13T14:46:55.167868+0000","last_clean":"2024-09-13T14:46:55.167868+0000","last_became_active":"2024-09-13T14:46:13.032612+0000","last_became_peered":"2024-09-13T14:46:13.032612+0000","last_unstale":"2024-09-13T14:46:55.167868+0000","last_undegraded":"2024-09-13T14:46:55.167868+0000","last_fullsized":"2024-09-13T14:46:55.167868+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:55.158893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"189'1038","reported_seq":1052,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.915993+0000","last_change":"2024-09-13T14:46:40.111886+0000","last_active":"2024-09-13T14:46:55.915993+0000","last_peered":"2024-09-13T14:46:55.915993+0000","last_clean":"2024-09-13T14:46:55.915993+0000","last_became_active":"2024-09-13T14:46:40.111569+0000","last_became_peered":"2024-09-13T14:46:40.111569+0000","last_unstale":"2024-09-13T14:46:55.915993+0000","last_undegraded":"2024-09-13T14:46:55.915993+0000","last_fullsized":"2024-09-13T14:46:55.915993+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1038,"log_dups_size":0,"ondisk_log_size":1038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:55.578734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68026368,"num_objects":1038,"num_object_clones":0,"num_object_copies":3114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1038,"num_write_kb":66432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":441,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288418+0000","last_change":"2024-09-13T14:46:16.101131+0000","last_active":"2024-09-13T14:46:49.288418+0000","last_peered":"2024-09-13T14:46:49.288418+0000","last_clean":"2024-09-13T14:46:49.288418+0000","last_became_active":"2024-09-13T14:46:16.100856+0000","last_became_peered":"2024-09-13T14:46:16.100856+0000","last_unstale":"2024-09-13T14:46:49.288418+0000","last_undegraded":"2024-09-13T14:46:49.288418+0000","last_fullsized":"2024-09-13T14:46:49.288418+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:23.900190+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288946+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:46:49.288946+0000","last_peered":"2024-09-13T14:46:49.288946+0000","last_clean":"2024-09-13T14:46:49.288946+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:46:49.288946+0000","last_undegraded":"2024-09-13T14:46:49.288946+0000","last_fullsized":"2024-09-13T14:46:49.288946+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167609+0000","last_change":"2024-09-13T14:46:13.037644+0000","last_active":"2024-09-13T14:46:55.167609+0000","last_peered":"2024-09-13T14:46:55.167609+0000","last_clean":"2024-09-13T14:46:55.167609+0000","last_became_active":"2024-09-13T14:46:13.034395+0000","last_became_peered":"2024-09-13T14:46:13.034395+0000","last_unstale":"2024-09-13T14:46:55.167609+0000","last_undegraded":"2024-09-13T14:46:55.167609+0000","last_fullsized":"2024-09-13T14:46:55.167609+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:22:05.467132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":210,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167130+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:46:55.167130+0000","last_peered":"2024-09-13T14:46:55.167130+0000","last_clean":"2024-09-13T14:46:55.167130+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:46:55.167130+0000","last_undegraded":"2024-09-13T14:46:55.167130+0000","last_fullsized":"2024-09-13T14:46:55.167130+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"189'1115","reported_seq":1129,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.341575+0000","last_change":"2024-09-13T14:46:40.217610+0000","last_active":"2024-09-13T14:46:56.341575+0000","last_peered":"2024-09-13T14:46:56.341575+0000","last_clean":"2024-09-13T14:46:56.341575+0000","last_became_active":"2024-09-13T14:46:40.217224+0000","last_became_peered":"2024-09-13T14:46:40.217224+0000","last_unstale":"2024-09-13T14:46:56.341575+0000","last_undegraded":"2024-09-13T14:46:56.341575+0000","last_fullsized":"2024-09-13T14:46:56.341575+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1115,"log_dups_size":0,"ondisk_log_size":1115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:26.853493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73072640,"num_objects":1115,"num_object_clones":0,"num_object_copies":3345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1115,"num_write_kb":71360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":432,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289557+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:46:49.289557+0000","last_peered":"2024-09-13T14:46:49.289557+0000","last_clean":"2024-09-13T14:46:49.289557+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:46:49.289557+0000","last_undegraded":"2024-09-13T14:46:49.289557+0000","last_fullsized":"2024-09-13T14:46:49.289557+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":171,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289028+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:46:49.289028+0000","last_peered":"2024-09-13T14:46:49.289028+0000","last_clean":"2024-09-13T14:46:49.289028+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:46:49.289028+0000","last_undegraded":"2024-09-13T14:46:49.289028+0000","last_fullsized":"2024-09-13T14:46:49.289028+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":255,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167260+0000","last_change":"2024-09-13T14:46:13.032370+0000","last_active":"2024-09-13T14:46:55.167260+0000","last_peered":"2024-09-13T14:46:55.167260+0000","last_clean":"2024-09-13T14:46:55.167260+0000","last_became_active":"2024-09-13T14:46:13.032017+0000","last_became_peered":"2024-09-13T14:46:13.032017+0000","last_unstale":"2024-09-13T14:46:55.167260+0000","last_undegraded":"2024-09-13T14:46:55.167260+0000","last_fullsized":"2024-09-13T14:46:55.167260+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:54:56.698830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167006+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:46:55.167006+0000","last_peered":"2024-09-13T14:46:55.167006+0000","last_clean":"2024-09-13T14:46:55.167006+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:46:55.167006+0000","last_undegraded":"2024-09-13T14:46:55.167006+0000","last_fullsized":"2024-09-13T14:46:55.167006+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.d","version":"189'1104","reported_seq":1116,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.914054+0000","last_change":"2024-09-13T14:46:40.114350+0000","last_active":"2024-09-13T14:46:55.914054+0000","last_peered":"2024-09-13T14:46:55.914054+0000","last_clean":"2024-09-13T14:46:55.914054+0000","last_became_active":"2024-09-13T14:46:40.114052+0000","last_became_peered":"2024-09-13T14:46:40.114052+0000","last_unstale":"2024-09-13T14:46:55.914054+0000","last_undegraded":"2024-09-13T14:46:55.914054+0000","last_fullsized":"2024-09-13T14:46:55.914054+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1104,"log_dups_size":0,"ondisk_log_size":1104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:58:53.903829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72351744,"num_objects":1104,"num_object_clones":0,"num_object_copies":3312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1104,"num_write_kb":70656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":449,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289149+0000","last_change":"2024-09-13T14:46:16.046664+0000","last_active":"2024-09-13T14:46:49.289149+0000","last_peered":"2024-09-13T14:46:49.289149+0000","last_clean":"2024-09-13T14:46:49.289149+0000","last_became_active":"2024-09-13T14:46:16.046293+0000","last_became_peered":"2024-09-13T14:46:16.046293+0000","last_unstale":"2024-09-13T14:46:49.289149+0000","last_undegraded":"2024-09-13T14:46:49.289149+0000","last_fullsized":"2024-09-13T14:46:49.289149+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:44:16.817130+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":197,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167223+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:46:55.167223+0000","last_peered":"2024-09-13T14:46:55.167223+0000","last_clean":"2024-09-13T14:46:55.167223+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:46:55.167223+0000","last_undegraded":"2024-09-13T14:46:55.167223+0000","last_fullsized":"2024-09-13T14:46:55.167223+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289611+0000","last_change":"2024-09-13T14:46:13.035683+0000","last_active":"2024-09-13T14:46:49.289611+0000","last_peered":"2024-09-13T14:46:49.289611+0000","last_clean":"2024-09-13T14:46:49.289611+0000","last_became_active":"2024-09-13T14:46:13.035300+0000","last_became_peered":"2024-09-13T14:46:13.035300+0000","last_unstale":"2024-09-13T14:46:49.289611+0000","last_undegraded":"2024-09-13T14:46:49.289611+0000","last_fullsized":"2024-09-13T14:46:49.289611+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:47.799849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":39,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288369+0000","last_change":"2024-09-13T14:46:13.032645+0000","last_active":"2024-09-13T14:46:49.288369+0000","last_peered":"2024-09-13T14:46:49.288369+0000","last_clean":"2024-09-13T14:46:49.288369+0000","last_became_active":"2024-09-13T14:46:13.031822+0000","last_became_peered":"2024-09-13T14:46:13.031822+0000","last_unstale":"2024-09-13T14:46:49.288369+0000","last_undegraded":"2024-09-13T14:46:49.288369+0000","last_fullsized":"2024-09-13T14:46:49.288369+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:27.907257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"189'1121","reported_seq":1135,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:56.356835+0000","last_change":"2024-09-13T14:46:40.115651+0000","last_active":"2024-09-13T14:46:56.356835+0000","last_peered":"2024-09-13T14:46:56.356835+0000","last_clean":"2024-09-13T14:46:56.356835+0000","last_became_active":"2024-09-13T14:46:40.115318+0000","last_became_peered":"2024-09-13T14:46:40.115318+0000","last_unstale":"2024-09-13T14:46:56.356835+0000","last_undegraded":"2024-09-13T14:46:56.356835+0000","last_fullsized":"2024-09-13T14:46:56.356835+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1121,"log_dups_size":0,"ondisk_log_size":1121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:09:34.766948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73465856,"num_objects":1121,"num_object_clones":0,"num_object_copies":3363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1121,"num_write_kb":71744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":424,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289387+0000","last_change":"2024-09-13T14:43:00.868011+0000","last_active":"2024-09-13T14:46:49.289387+0000","last_peered":"2024-09-13T14:46:49.289387+0000","last_clean":"2024-09-13T14:46:49.289387+0000","last_became_active":"2024-09-13T14:42:56.283775+0000","last_became_peered":"2024-09-13T14:42:56.283775+0000","last_unstale":"2024-09-13T14:46:49.289387+0000","last_undegraded":"2024-09-13T14:46:49.289387+0000","last_fullsized":"2024-09-13T14:46:49.289387+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:00.867870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:00.867870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:25.806175+0000","scrub_duration":0.0043830680000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":197,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.170291+0000","last_change":"2024-09-13T14:46:13.042846+0000","last_active":"2024-09-13T14:46:55.170291+0000","last_peered":"2024-09-13T14:46:55.170291+0000","last_clean":"2024-09-13T14:46:55.170291+0000","last_became_active":"2024-09-13T14:46:13.035242+0000","last_became_peered":"2024-09-13T14:46:13.035242+0000","last_unstale":"2024-09-13T14:46:55.170291+0000","last_undegraded":"2024-09-13T14:46:55.170291+0000","last_fullsized":"2024-09-13T14:46:55.170291+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:25:51.901487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168290+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:46:55.168290+0000","last_peered":"2024-09-13T14:46:55.168290+0000","last_clean":"2024-09-13T14:46:55.168290+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:46:55.168290+0000","last_undegraded":"2024-09-13T14:46:55.168290+0000","last_fullsized":"2024-09-13T14:46:55.168290+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166383+0000","last_change":"2024-09-13T14:46:13.037280+0000","last_active":"2024-09-13T14:46:55.166383+0000","last_peered":"2024-09-13T14:46:55.166383+0000","last_clean":"2024-09-13T14:46:55.166383+0000","last_became_active":"2024-09-13T14:46:13.033562+0000","last_became_peered":"2024-09-13T14:46:13.033562+0000","last_unstale":"2024-09-13T14:46:55.166383+0000","last_undegraded":"2024-09-13T14:46:55.166383+0000","last_fullsized":"2024-09-13T14:46:55.166383+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:41:51.627266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"189'1051","reported_seq":1065,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.900736+0000","last_change":"2024-09-13T14:46:40.112937+0000","last_active":"2024-09-13T14:46:55.900736+0000","last_peered":"2024-09-13T14:46:55.900736+0000","last_clean":"2024-09-13T14:46:55.900736+0000","last_became_active":"2024-09-13T14:46:40.112673+0000","last_became_peered":"2024-09-13T14:46:40.112673+0000","last_unstale":"2024-09-13T14:46:55.900736+0000","last_undegraded":"2024-09-13T14:46:55.900736+0000","last_fullsized":"2024-09-13T14:46:55.900736+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":184,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:46:39.095656+0000","last_clean_scrub_stamp":"2024-09-13T14:46:39.095656+0000","objects_scrubbed":0,"log_size":1051,"log_dups_size":0,"ondisk_log_size":1051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:21:54.396577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1051,"num_write_kb":67264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":458,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168775+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:46:55.168775+0000","last_peered":"2024-09-13T14:46:55.168775+0000","last_clean":"2024-09-13T14:46:55.168775+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:46:55.168775+0000","last_undegraded":"2024-09-13T14:46:55.168775+0000","last_fullsized":"2024-09-13T14:46:55.168775+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":62,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168867+0000","last_change":"2024-09-13T14:46:13.044728+0000","last_active":"2024-09-13T14:46:55.168867+0000","last_peered":"2024-09-13T14:46:55.168867+0000","last_clean":"2024-09-13T14:46:55.168867+0000","last_became_active":"2024-09-13T14:46:13.036980+0000","last_became_peered":"2024-09-13T14:46:13.036980+0000","last_unstale":"2024-09-13T14:46:55.168867+0000","last_undegraded":"2024-09-13T14:46:55.168867+0000","last_fullsized":"2024-09-13T14:46:55.168867+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:50:41.200194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":169,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289786+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:46:49.289786+0000","last_peered":"2024-09-13T14:46:49.289786+0000","last_clean":"2024-09-13T14:46:49.289786+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:46:49.289786+0000","last_undegraded":"2024-09-13T14:46:49.289786+0000","last_fullsized":"2024-09-13T14:46:49.289786+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289737+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:46:49.289737+0000","last_peered":"2024-09-13T14:46:49.289737+0000","last_clean":"2024-09-13T14:46:49.289737+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:46:49.289737+0000","last_undegraded":"2024-09-13T14:46:49.289737+0000","last_fullsized":"2024-09-13T14:46:49.289737+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289403+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:46:49.289403+0000","last_peered":"2024-09-13T14:46:49.289403+0000","last_clean":"2024-09-13T14:46:49.289403+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:46:49.289403+0000","last_undegraded":"2024-09-13T14:46:49.289403+0000","last_fullsized":"2024-09-13T14:46:49.289403+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":164,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.290301+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:46:49.290301+0000","last_peered":"2024-09-13T14:46:49.290301+0000","last_clean":"2024-09-13T14:46:49.290301+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:46:49.290301+0000","last_undegraded":"2024-09-13T14:46:49.290301+0000","last_fullsized":"2024-09-13T14:46:49.290301+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":118,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167211+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:46:55.167211+0000","last_peered":"2024-09-13T14:46:55.167211+0000","last_clean":"2024-09-13T14:46:55.167211+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:46:55.167211+0000","last_undegraded":"2024-09-13T14:46:55.167211+0000","last_fullsized":"2024-09-13T14:46:55.167211+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":143,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.171806+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:46:55.171806+0000","last_peered":"2024-09-13T14:46:55.171806+0000","last_clean":"2024-09-13T14:46:55.171806+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:46:55.171806+0000","last_undegraded":"2024-09-13T14:46:55.171806+0000","last_fullsized":"2024-09-13T14:46:55.171806+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":154,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288636+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:46:49.288636+0000","last_peered":"2024-09-13T14:46:49.288636+0000","last_clean":"2024-09-13T14:46:49.288636+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:46:49.288636+0000","last_undegraded":"2024-09-13T14:46:49.288636+0000","last_fullsized":"2024-09-13T14:46:49.288636+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":185,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166386+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:46:55.166386+0000","last_peered":"2024-09-13T14:46:55.166386+0000","last_clean":"2024-09-13T14:46:55.166386+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:46:55.166386+0000","last_undegraded":"2024-09-13T14:46:55.166386+0000","last_fullsized":"2024-09-13T14:46:55.166386+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289049+0000","last_change":"2024-09-13T14:46:13.036225+0000","last_active":"2024-09-13T14:46:49.289049+0000","last_peered":"2024-09-13T14:46:49.289049+0000","last_clean":"2024-09-13T14:46:49.289049+0000","last_became_active":"2024-09-13T14:46:13.035435+0000","last_became_peered":"2024-09-13T14:46:13.035435+0000","last_unstale":"2024-09-13T14:46:49.289049+0000","last_undegraded":"2024-09-13T14:46:49.289049+0000","last_fullsized":"2024-09-13T14:46:49.289049+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:13.907280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167142+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:46:55.167142+0000","last_peered":"2024-09-13T14:46:55.167142+0000","last_clean":"2024-09-13T14:46:55.167142+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:46:55.167142+0000","last_undegraded":"2024-09-13T14:46:55.167142+0000","last_fullsized":"2024-09-13T14:46:55.167142+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289634+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:46:49.289634+0000","last_peered":"2024-09-13T14:46:49.289634+0000","last_clean":"2024-09-13T14:46:49.289634+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:46:49.289634+0000","last_undegraded":"2024-09-13T14:46:49.289634+0000","last_fullsized":"2024-09-13T14:46:49.289634+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":221,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166835+0000","last_change":"2024-09-13T14:46:13.037050+0000","last_active":"2024-09-13T14:46:55.166835+0000","last_peered":"2024-09-13T14:46:55.166835+0000","last_clean":"2024-09-13T14:46:55.166835+0000","last_became_active":"2024-09-13T14:46:13.033971+0000","last_became_peered":"2024-09-13T14:46:13.033971+0000","last_unstale":"2024-09-13T14:46:55.166835+0000","last_undegraded":"2024-09-13T14:46:55.166835+0000","last_fullsized":"2024-09-13T14:46:55.166835+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:26:52.030341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":142,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168411+0000","last_change":"2024-09-13T14:46:13.036470+0000","last_active":"2024-09-13T14:46:55.168411+0000","last_peered":"2024-09-13T14:46:55.168411+0000","last_clean":"2024-09-13T14:46:55.168411+0000","last_became_active":"2024-09-13T14:46:13.036136+0000","last_became_peered":"2024-09-13T14:46:13.036136+0000","last_unstale":"2024-09-13T14:46:55.168411+0000","last_undegraded":"2024-09-13T14:46:55.168411+0000","last_fullsized":"2024-09-13T14:46:55.168411+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:27:50.210031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":196,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167414+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:46:55.167414+0000","last_peered":"2024-09-13T14:46:55.167414+0000","last_clean":"2024-09-13T14:46:55.167414+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:46:55.167414+0000","last_undegraded":"2024-09-13T14:46:55.167414+0000","last_fullsized":"2024-09-13T14:46:55.167414+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":198,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.169767+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:46:55.169767+0000","last_peered":"2024-09-13T14:46:55.169767+0000","last_clean":"2024-09-13T14:46:55.169767+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:46:55.169767+0000","last_undegraded":"2024-09-13T14:46:55.169767+0000","last_fullsized":"2024-09-13T14:46:55.169767+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":122,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166590+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:46:55.166590+0000","last_peered":"2024-09-13T14:46:55.166590+0000","last_clean":"2024-09-13T14:46:55.166590+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:46:55.166590+0000","last_undegraded":"2024-09-13T14:46:55.166590+0000","last_fullsized":"2024-09-13T14:46:55.166590+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":223,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.171873+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:46:55.171873+0000","last_peered":"2024-09-13T14:46:55.171873+0000","last_clean":"2024-09-13T14:46:55.171873+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:46:55.171873+0000","last_undegraded":"2024-09-13T14:46:55.171873+0000","last_fullsized":"2024-09-13T14:46:55.171873+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":148,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.167420+0000","last_change":"2024-09-13T14:46:13.039825+0000","last_active":"2024-09-13T14:46:55.167420+0000","last_peered":"2024-09-13T14:46:55.167420+0000","last_clean":"2024-09-13T14:46:55.167420+0000","last_became_active":"2024-09-13T14:46:13.037216+0000","last_became_peered":"2024-09-13T14:46:13.037216+0000","last_unstale":"2024-09-13T14:46:55.167420+0000","last_undegraded":"2024-09-13T14:46:55.167420+0000","last_fullsized":"2024-09-13T14:46:55.167420+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:08:33.966564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":169,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289331+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:46:49.289331+0000","last_peered":"2024-09-13T14:46:49.289331+0000","last_clean":"2024-09-13T14:46:49.289331+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:46:49.289331+0000","last_undegraded":"2024-09-13T14:46:49.289331+0000","last_fullsized":"2024-09-13T14:46:49.289331+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":156,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288729+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:46:49.288729+0000","last_peered":"2024-09-13T14:46:49.288729+0000","last_clean":"2024-09-13T14:46:49.288729+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:46:49.288729+0000","last_undegraded":"2024-09-13T14:46:49.288729+0000","last_fullsized":"2024-09-13T14:46:49.288729+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":178,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289934+0000","last_change":"2024-09-13T14:46:13.030959+0000","last_active":"2024-09-13T14:46:49.289934+0000","last_peered":"2024-09-13T14:46:49.289934+0000","last_clean":"2024-09-13T14:46:49.289934+0000","last_became_active":"2024-09-13T14:46:13.030630+0000","last_became_peered":"2024-09-13T14:46:13.030630+0000","last_unstale":"2024-09-13T14:46:49.289934+0000","last_undegraded":"2024-09-13T14:46:49.289934+0000","last_fullsized":"2024-09-13T14:46:49.289934+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:21.669809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":141,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.288971+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:46:49.288971+0000","last_peered":"2024-09-13T14:46:49.288971+0000","last_clean":"2024-09-13T14:46:49.288971+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:46:49.288971+0000","last_undegraded":"2024-09-13T14:46:49.288971+0000","last_fullsized":"2024-09-13T14:46:49.288971+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":237,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.166769+0000","last_change":"2024-09-13T14:46:13.034696+0000","last_active":"2024-09-13T14:46:55.166769+0000","last_peered":"2024-09-13T14:46:55.166769+0000","last_clean":"2024-09-13T14:46:55.166769+0000","last_became_active":"2024-09-13T14:46:13.034373+0000","last_became_peered":"2024-09-13T14:46:13.034373+0000","last_unstale":"2024-09-13T14:46:55.166769+0000","last_undegraded":"2024-09-13T14:46:55.166769+0000","last_fullsized":"2024-09-13T14:46:55.166769+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:40:58.170678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":220,"reported_epoch":189,"state":"active+clean","last_fresh":"2024-09-13T14:46:55.168583+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:46:55.168583+0000","last_peered":"2024-09-13T14:46:55.168583+0000","last_clean":"2024-09-13T14:46:55.168583+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:46:55.168583+0000","last_undegraded":"2024-09-13T14:46:55.168583+0000","last_fullsized":"2024-09-13T14:46:55.168583+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":65,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289037+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:46:49.289037+0000","last_peered":"2024-09-13T14:46:49.289037+0000","last_clean":"2024-09-13T14:46:49.289037+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:46:49.289037+0000","last_undegraded":"2024-09-13T14:46:49.289037+0000","last_fullsized":"2024-09-13T14:46:49.289037+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":51,"reported_epoch":188,"state":"active+clean","last_fresh":"2024-09-13T14:46:49.289047+0000","last_change":"2024-09-13T14:46:13.040698+0000","last_active":"2024-09-13T14:46:49.289047+0000","last_peered":"2024-09-13T14:46:49.289047+0000","last_clean":"2024-09-13T14:46:49.289047+0000","last_became_active":"2024-09-13T14:46:13.029766+0000","last_became_peered":"2024-09-13T14:46:13.029766+0000","last_unstale":"2024-09-13T14:46:49.289047+0000","last_undegraded":"2024-09-13T14:46:49.289047+0000","last_fullsized":"2024-09-13T14:46:49.289047+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:37:59.121297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1131806720,"num_objects":17270,"num_object_clones":0,"num_object_copies":51810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17270,"num_write_kb":1105280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3499950080,"data_stored":3499950080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17270,"ondisk_log_size":17270,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":694,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234322,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1066340,"kb_used_data":367448,"kb_used_omap":1,"kb_used_meta":698878,"kb_avail":92678812,"statfs":{"total":95995035648,"available":94903103488,"internally_reserved":0,"allocated":376266752,"data_stored":375336317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":715651531},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568495960,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1008416,"kb_used_data":471960,"kb_used_omap":1,"kb_used_meta":536446,"kb_avail":92736736,"statfs":{"total":95995035648,"available":94962417664,"internally_reserved":0,"allocated":483287040,"data_stored":482357150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":549321159},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757598,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173272,"kb_used_data":366092,"kb_used_omap":1,"kb_used_meta":807166,"kb_avail":92571880,"statfs":{"total":95995035648,"available":94793605120,"internally_reserved":0,"allocated":374878208,"data_stored":373956063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":826538438},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019234,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1293280,"kb_used_data":644436,"kb_used_omap":1,"kb_used_meta":648830,"kb_avail":92451872,"statfs":{"total":95995035648,"available":94670716928,"internally_reserved":0,"allocated":659902464,"data_stored":658976111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":664402374},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650446,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767188,"kb_used_data":520584,"kb_used_omap":4,"kb_used_meta":246587,"kb_avail":92977964,"statfs":{"total":95995035648,"available":95209435136,"internally_reserved":0,"allocated":533078016,"data_stored":532159729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":252505647},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879381,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":536588,"kb_used_data":407172,"kb_used_omap":6,"kb_used_meta":129401,"kb_avail":93208564,"statfs":{"total":95995035648,"available":95445569536,"internally_reserved":0,"allocated":416944128,"data_stored":416022119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":132507193},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435398,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944268,"kb_used_data":541444,"kb_used_omap":5,"kb_used_meta":402810,"kb_avail":92800884,"statfs":{"total":95995035648,"available":95028105216,"internally_reserved":0,"allocated":554438656,"data_stored":553523360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":412477793},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":403726925972,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":560463872,"data_stored":560463872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":443351040,"data_stored":443351040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":538247168,"data_stored":538247168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":666566656,"data_stored":666566656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":424411136,"data_stored":424411136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":481886208,"data_stored":481886208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":385024000,"data_stored":385024000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:47:04.228 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:47:04.228 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-09-13T14:47:04.249 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:04.257 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:04.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:04.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:04.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:04.477 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:04 smithi042 ceph-mon[108570]: pgmap v668: 99 pgs: 99 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-13T14:47:04.478 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:04 smithi042 ceph-mon[113926]: pgmap v668: 99 pgs: 99 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-13T14:47:04.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:04 smithi159 ceph-mon[102519]: pgmap v668: 99 pgs: 99 active+clean; 1.8 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 181 MiB/s wr, 2.90k op/s 2024-09-13T14:47:04.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:04.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:04.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:04.661 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:04.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:04.797 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:04.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:04.931 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:04.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:04.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:05.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.069 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 38162 38146 170.273 122.25 0.00500489 0.00586715 2024-09-13T14:47:05.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:05.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:05.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:05.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:05.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:05.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:05.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:05.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:05.605 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:05.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:05.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:05.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:05.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:05.842 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:05.882 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:05.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:05.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:06.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:06.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 39836 39820 165.896 104.625 0.00448109 0.00602339 2024-09-13T14:47:06.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:06.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:06.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:06.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:06.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:06.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[108570]: pgmap v669: 99 pgs: 99 active+clean; 1.8 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 182 MiB/s wr, 2.91k op/s 2024-09-13T14:47:06.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:06.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.492 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[113926]: pgmap v669: 99 pgs: 99 active+clean; 1.8 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 182 MiB/s wr, 2.91k op/s 2024-09-13T14:47:06.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:06.493 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:06 smithi159 ceph-mon[102519]: pgmap v669: 99 pgs: 99 active+clean; 1.8 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 182 MiB/s wr, 2.91k op/s 2024-09-13T14:47:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:06.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:06.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:06.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:06.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:06.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:06.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:06.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:06.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:06.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:07.002 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:07.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 41445 41429 161.811 100.562 0.00328904 0.00616662 2024-09-13T14:47:07.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:07.133 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:07.235 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:07.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:07.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:07.381 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:07.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:07.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:07.540 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:47:07.541 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:07.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:07.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:07.572 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:47:07.573 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:07.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:47:07.602 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:07.638 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:47:07.638 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:07.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:07.674 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:47:07.675 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.701 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:07.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:07.710 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:47:07.710 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:07.744 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:47:07.744 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:07.775 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:47:07.775 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41235 10.4 10:2c60add5:::benchmark_data_smithi042_191918_object41234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.951785+0000", 2024-09-13T14:47:07.789 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.013096234999999999, 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013127389, 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41235 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951785+0000", 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.790 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951785+0000", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951785+0000", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951815+0000", 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9899999999999998e-05 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951821+0000", 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3199999999999996e-06 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.951826+0000", 2024-09-13T14:47:07.792 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4220000000000002e-06 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.959771+0000", 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0079452329999999995 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.959883+0000", 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112742 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:47:07.793 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.960007+0000", 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012362899999999999 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961813+0000", 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0018057279999999999 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.962381+0000", 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00056862199999999996 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.794 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41244 10.1 10:8cda4ae4:::benchmark_data_smithi042_191918_object41243:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.958562+0000", 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0063196700000000003, 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.006437155, 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41244 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.795 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958562+0000", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958562+0000", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958561+0000", 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958593+0000", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1571e-05 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958600+0000", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6019999999999996e-06 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.958604+0000", 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.515e-06 2024-09-13T14:47:07.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.960892+0000", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022874700000000002 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961018+0000", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012618899999999999 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961148+0000", 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013032500000000001 2024-09-13T14:47:07.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963729+0000", 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00258114 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963745+0000", 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5928000000000001e-05 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.799 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:41238 10.b e189/184 10:d1b1a9e4:::benchmark_data_smithi042_191918_object41237:head v 189'2633, mlcod=189'2629)", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.961543+0000", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0033390339999999998, 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035319090000000002, 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961543+0000", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961543+0000", 2024-09-13T14:47:07.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961542+0000", 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961567+0000", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4538000000000001e-05 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961573+0000", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4160000000000001e-06 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961578+0000", 2024-09-13T14:47:07.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8339999999999998e-06 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963287+0000", 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001708698 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963308+0000", 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.145e-05 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.803 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:41242 10.d e189/184 10:bb97c52a:::benchmark_data_smithi042_191918_object41241:head v 189'2692, mlcod=189'2690)", 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.961613+0000", 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.003268174, 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035154589999999999, 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961613+0000", 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961613+0000", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961613+0000", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961637+0000", 2024-09-13T14:47:07.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4054999999999999e-05 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961645+0000", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7770000000000002e-06 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961650+0000", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2279999999999998e-06 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961691+0000", 2024-09-13T14:47:07.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0717999999999999e-05 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.961709+0000", 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8284e-05 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:41239 10.0 e189/184 10:09c0ec51:::benchmark_data_smithi042_191918_object41238:head v 189'2580, mlcod=189'2577)", 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.963288+0000", 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0015932679999999999, 2024-09-13T14:47:07.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0018916499999999999, 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963288+0000", 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963288+0000", 2024-09-13T14:47:07.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963288+0000", 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963318+0000", 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0562999999999999e-05 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.809 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963426+0000", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000108137 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963433+0000", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963475+0000", 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2259000000000003e-05 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.810 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.963493+0000", 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.783e-05 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41245 10.1 10:82e5c813:::benchmark_data_smithi042_191918_object41244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.964040+0000", 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00084202599999999995, 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011972490000000001, 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.811 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41245 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964040+0000", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964040+0000", 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964038+0000", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964084+0000", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5657e-05 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964101+0000", 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6714e-05 2024-09-13T14:47:07.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964109+0000", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1580000000000005e-06 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964371+0000", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00026278299999999999 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964533+0000", 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016137200000000001 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964697+0000", 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016424500000000001 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41248 10.e 10:7850fda9:::benchmark_data_smithi042_191918_object41247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.964120+0000", 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.000761161, 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011827369999999999, 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.815 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41248 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964120+0000", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964120+0000", 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964120+0000", 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964157+0000", 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6845999999999999e-05 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964165+0000", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7819999999999999e-06 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964171+0000", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8420000000000001e-06 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964560+0000", 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00038877900000000001 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.818 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964694+0000", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000134101 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964833+0000", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013877099999999999 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41249 10.e 10.21ef484e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.964183+0000", 2024-09-13T14:47:07.819 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.000698651, 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011841880000000001, 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41249 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964183+0000", 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964183+0000", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964182+0000", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964214+0000", 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1714000000000002e-05 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964222+0000", 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7570000000000004e-06 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.964227+0000", 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4129999999999997e-06 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41252 10.c 10:338e8a36:::benchmark_data_smithi042_191918_object41251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.822 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.965021+0000", 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9998608, 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039821099999999999, 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41252 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.823 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965021+0000", 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965021+0000", 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965020+0000", 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965055+0000", 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4853e-05 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965067+0000", 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2351000000000001e-05 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965074+0000", 2024-09-13T14:47:07.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3889999999999999e-06 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965257+0000", 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018308899999999999 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965363+0000", 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106074 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.827 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:41253 10.e 10.640155fe (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e189)", 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.965089+0000", 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9997926, 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00038634200000000002, 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:07.828 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 41253 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965089+0000", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:07.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965089+0000", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965089+0000", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965117+0000", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8518e-05 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965124+0000", 2024-09-13T14:47:07.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0960000000000003e-06 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.965130+0000", 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7950000000000004e-06 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 10 2024-09-13T14:47:07.831 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:47:07.831 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:07.866 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:07.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:07.967 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:08.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 43150 43134 158.559 106.562 0.00815721 0.00630027 2024-09-13T14:47:08.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:08.143 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:08.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:08.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:08 smithi042 ceph-mon[113926]: pgmap v670: 99 pgs: 99 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 202 MiB/s wr, 3.24k op/s 2024-09-13T14:47:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/715256746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:08 smithi042 ceph-mon[108570]: pgmap v670: 99 pgs: 99 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 202 MiB/s wr, 3.24k op/s 2024-09-13T14:47:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/715256746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:08.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:08.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:08.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:08 smithi159 ceph-mon[102519]: pgmap v670: 99 pgs: 99 active+clean; 2.2 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 202 MiB/s wr, 3.24k op/s 2024-09-13T14:47:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/715256746' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:08.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:08.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:08.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.663 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:08.695 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:08.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:08.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:08.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:08.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:08.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:08.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:09.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:09.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 44863 44847 155.698 107.062 0.0079407 0.00641688 2024-09-13T14:47:09.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:09.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:09.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:09.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:09.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:09.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:09.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:09.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:09.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:09.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:47:09.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:09.634 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:09.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:09.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:09.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.781 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:09.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:09.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:09.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:09.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:10.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:10.061 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:10.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:10.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 46525 46509 152.97 103.875 0.00767958 0.00653278 2024-09-13T14:47:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:09] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-13T14:47:10.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:10.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:10.214 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:47:10.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:10.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:10.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:10.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:10.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:10 smithi159 ceph-mon[102519]: pgmap v671: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-09-13T14:47:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:10 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:10.592 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:10.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[108570]: pgmap v671: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-09-13T14:47:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[113926]: pgmap v671: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-09-13T14:47:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:10.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:10.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:10.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:10.841 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:10.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:10.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:10.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:11.080 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:11.095 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:47:11.095867+0000 min lat: 0.0013466 max lat: 0.138001 avg lat: 0.00667469 2024-09-13T14:47:11.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:47:11.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 47922 47906 149.687 87.3125 0.00510833 0.00667469 2024-09-13T14:47:11.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:11.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:11.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:11.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:11.357 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:11.391 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:11.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:11 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:11.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:11 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:11.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:11 smithi159 ceph-mon[102519]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:47:11.530 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:11.530 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:11.530 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[113926]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:47:11.530 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:11.531 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3359343187' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:11.531 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:11 smithi042 ceph-mon[108570]: osdmap e190: 8 total, 8 up, 7 in 2024-09-13T14:47:11.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:11.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:11.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:11.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:11.773 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:11.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:11.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:11.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:11.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:12.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:12.078 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:12.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 49483 49467 147.205 97.5625 0.066609 0.00678493 2024-09-13T14:47:12.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:12.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:12.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:12.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:12.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:12.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:12.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:12.469 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:12 smithi042 ceph-mon[113926]: pgmap v673: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:47:12.470 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:12 smithi042 ceph-mon[108570]: pgmap v673: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:47:12.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:12.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:12 smithi159 ceph-mon[102519]: pgmap v673: 99 pgs: 99 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-09-13T14:47:12.512 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:12.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:12.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:12.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:12.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:12.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:12.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:12.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:12.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:12.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:13.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 51034 51018 144.919 96.9375 0.00766501 0.0068956 2024-09-13T14:47:13.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:13.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:13.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:47:13.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:13.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:13.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:13.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:13 smithi159 ceph-mon[102519]: pgmap v674: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:47:13.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:13.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:13 smithi042 ceph-mon[108570]: pgmap v674: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:47:13.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:13 smithi042 ceph-mon[113926]: pgmap v674: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-09-13T14:47:13.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:13.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:13.624 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:13.668 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:13.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:13.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:13.851 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:13.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:13.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:13.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:14.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:14.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:14.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 52299 52283 142.056 79.0625 0.00810064 0.00703472 2024-09-13T14:47:14.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:14.152 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:14.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:14.282 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:14.289 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 10 pg_num to 26 2024-09-13T14:47:14.305 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3296184193' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.305 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.305 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3296184193' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:14.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:14.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3296184193' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:47:14.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:14.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:14.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:14.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:14.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:14.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:14.944 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:14.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:14.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:14.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:15.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:15.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 53619 53603 139.574 82.5 0.00819568 0.00715897 2024-09-13T14:47:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.140 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:15.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:15.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:15.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:15.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:15.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:15.440 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:15.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:15.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: pgmap v676: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:47:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:47:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: pgmap v676: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-13T14:47:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:15.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:15.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:15.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:47:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: osdmap e191: 8 total, 8 up, 7 in 2024-09-13T14:47:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: pgmap v676: 99 pgs: 99 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-13T14:47:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:47:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:15.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:15.823 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:15.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:15.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:15.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:15.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:15.998 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:47:16.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:16.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:16.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:16.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 53883 53867 134.651 16.5 0.00269887 0.00720725 2024-09-13T14:47:16.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:16.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:16.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:16.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:16.321 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:47:16.321 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:16 smithi042 ceph-mon[113926]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:47:16.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:47:16.324 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:16 smithi042 ceph-mon[108570]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:47:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:16.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:16.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:16.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:16.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:16.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:16.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:16.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:16.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:47:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:16 smithi159 ceph-mon[102519]: osdmap e192: 8 total, 8 up, 7 in 2024-09-13T14:47:16.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:16.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:16.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:16.993 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:17.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:17.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:17.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 55244 55228 132.744 85.0625 0.00568629 0.00752838 2024-09-13T14:47:17.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.174 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:17.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:17.274 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:17.310 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:17.412 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:17.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:17.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:17.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:17.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[113926]: pgmap v678: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:47:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[113926]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:47:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[113926]: 10.4 scrub starts 2024-09-13T14:47:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[113926]: 10.5 scrub starts 2024-09-13T14:47:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[108570]: pgmap v678: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:47:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[108570]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:47:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[108570]: 10.4 scrub starts 2024-09-13T14:47:17.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:17 smithi042 ceph-mon[108570]: 10.5 scrub starts 2024-09-13T14:47:17.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:17 smithi159 ceph-mon[102519]: pgmap v678: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-13T14:47:17.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:17 smithi159 ceph-mon[102519]: osdmap e193: 8 total, 8 up, 7 in 2024-09-13T14:47:17.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:17 smithi159 ceph-mon[102519]: 10.4 scrub starts 2024-09-13T14:47:17.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:17 smithi159 ceph-mon[102519]: 10.5 scrub starts 2024-09-13T14:47:17.695 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:17.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:17.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:17.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:17.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:17.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:17.905 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:17.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:18.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:18.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 57046 57030 131.998 112.625 0.00885174 0.00757065 2024-09-13T14:47:18.105 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:18.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:18.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:18.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:18.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:18.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:18.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.589 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:18.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[108570]: 10.4 scrub ok 2024-09-13T14:47:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[108570]: 10.2 scrub starts 2024-09-13T14:47:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[108570]: 10.5 scrub ok 2024-09-13T14:47:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[108570]: 10.8 scrub starts 2024-09-13T14:47:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[113926]: 10.4 scrub ok 2024-09-13T14:47:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[113926]: 10.2 scrub starts 2024-09-13T14:47:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[113926]: 10.5 scrub ok 2024-09-13T14:47:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:18 smithi042 ceph-mon[113926]: 10.8 scrub starts 2024-09-13T14:47:18.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:47:18.720 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:18.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:18.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:18 smithi159 ceph-mon[102519]: 10.4 scrub ok 2024-09-13T14:47:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:18 smithi159 ceph-mon[102519]: 10.2 scrub starts 2024-09-13T14:47:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:18 smithi159 ceph-mon[102519]: 10.5 scrub ok 2024-09-13T14:47:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:18 smithi159 ceph-mon[102519]: 10.8 scrub starts 2024-09-13T14:47:18.749 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:47:18.749 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:18.777 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:47:18.777 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:18.805 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:47:18.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:18.829 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:18.841 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:47:18.841 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:18.872 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:47:18.872 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:18.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:18.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:18.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:18.904 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:47:18.904 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:18.935 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:47:18.936 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:18.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:19.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:19.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 58744 58728 131.074 106.125 0.00542373 0.00762389 2024-09-13T14:47:19.117 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.155 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.257 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.360 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:47:19.399 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:19.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:19.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.578 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: 10.2 scrub ok 2024-09-13T14:47:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: 10.12 scrub starts 2024-09-13T14:47:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: pgmap v680: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: 10.1 scrub starts 2024-09-13T14:47:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: 10.8 scrub ok 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2732681443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1930545685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[113926]: 10.3 scrub starts 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: 10.2 scrub ok 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: 10.12 scrub starts 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: pgmap v680: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: 10.1 scrub starts 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: 10.8 scrub ok 2024-09-13T14:47:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2732681443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1930545685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:19.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-mon[108570]: 10.3 scrub starts 2024-09-13T14:47:19.679 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:19.706 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: 10.2 scrub ok 2024-09-13T14:47:19.706 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: 10.12 scrub starts 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: pgmap v680: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 51 MiB/s wr, 808 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: 10.1 scrub starts 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: 10.8 scrub ok 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2732681443' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1930545685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:19.707 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:19 smithi159 ceph-mon[102519]: 10.3 scrub starts 2024-09-13T14:47:19.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:19.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:19.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:19.956 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:19.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:19.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:19.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:20.029 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:19] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.51.0" 2024-09-13T14:47:20.091 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:20.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 60489 60473 130.315 109.062 0.0091139 0.00766929 2024-09-13T14:47:20.121 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:20.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:20.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:20.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:20.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:20.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:20.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:20.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:20.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: 10.12 scrub ok 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: 10.1 scrub ok 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: 10.0 scrub starts 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: 10.3 scrub ok 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[108570]: 10.15 scrub starts 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: 10.12 scrub ok 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: 10.1 scrub ok 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: 10.0 scrub starts 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: 10.3 scrub ok 2024-09-13T14:47:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:20 smithi042 ceph-mon[113926]: 10.15 scrub starts 2024-09-13T14:47:20.620 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: 10.12 scrub ok 2024-09-13T14:47:20.620 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: 10.1 scrub ok 2024-09-13T14:47:20.620 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: 10.0 scrub starts 2024-09-13T14:47:20.620 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:47:20.620 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:47:20.621 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:20.621 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: osdmap e194: 8 total, 8 up, 7 in 2024-09-13T14:47:20.621 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: 10.3 scrub ok 2024-09-13T14:47:20.621 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:20 smithi159 ceph-mon[102519]: 10.15 scrub starts 2024-09-13T14:47:20.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:20.696 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:20.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:20.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:20.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:20.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:20.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:20.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:20.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:20.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:21.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:21.096 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 62085 62069 129.296 99.75 0.00632552 0.00772926 2024-09-13T14:47:21.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:21.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:21.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:21.240 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:21.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:21.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:21.371 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:21.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:21.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: 10.0 scrub ok 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: pgmap v682: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: 10.15 scrub ok 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:21.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[113926]: 10.13 scrub starts 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: 10.0 scrub ok 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: pgmap v682: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: 10.15 scrub ok 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:21.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:21 smithi042 ceph-mon[108570]: 10.13 scrub starts 2024-09-13T14:47:21.611 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:21.632 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:21.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:21.734 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: 10.0 scrub ok 2024-09-13T14:47:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: pgmap v682: 109 pgs: 1 active, 1 active+clean+scrubbing, 7 peering, 7 unknown, 93 active+clean; 3.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 724 op/s; 11 KiB/s, 0 objects/s recovering 2024-09-13T14:47:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: 10.15 scrub ok 2024-09-13T14:47:21.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:21.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:21.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:21 smithi159 ceph-mon[102519]: 10.13 scrub starts 2024-09-13T14:47:21.769 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:21.872 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:21.911 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:21.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:21.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:22.030 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:47:22.030 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:47:22.030 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-09-13T14:47:22.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-09-13T14:47:22.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.040 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:22.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:22.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 63729 63713 128.439 102.75 0.00459558 0.00778139 2024-09-13T14:47:22.141 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:22.178 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:22.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:22.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:22.311 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:22.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:22.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:22.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:22.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:47:22.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:22.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:22.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:22.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:22.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:22.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:22 smithi159 ceph-mon[102519]: 10.14 scrub starts 2024-09-13T14:47:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:22 smithi159 ceph-mon[102519]: 10.14 scrub ok 2024-09-13T14:47:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:22 smithi159 ceph-mon[102519]: 10.13 scrub ok 2024-09-13T14:47:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:22 smithi159 ceph-mon[102519]: 10.18 scrub starts 2024-09-13T14:47:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:47:22.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:22.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[113926]: 10.14 scrub starts 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[113926]: 10.14 scrub ok 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[113926]: 10.13 scrub ok 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[113926]: 10.18 scrub starts 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:47:22.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[108570]: 10.14 scrub starts 2024-09-13T14:47:22.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[108570]: 10.14 scrub ok 2024-09-13T14:47:22.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[108570]: 10.13 scrub ok 2024-09-13T14:47:22.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[108570]: 10.18 scrub starts 2024-09-13T14:47:22.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:47:22.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:22.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:22.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:22.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:23.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:23.076 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:23.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 64398 64382 125.732 41.8125 0.00240032 0.00784105 2024-09-13T14:47:23.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:23.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:23.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:23.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:23.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:23.451 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:23.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:23.578 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:23.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[113926]: 10.6 scrub starts 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[113926]: pgmap v683: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.1 KiB/s, 0 objects/s recovering 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[113926]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[113926]: 10.18 scrub ok 2024-09-13T14:47:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[108570]: 10.6 scrub starts 2024-09-13T14:47:23.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[108570]: pgmap v683: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.1 KiB/s, 0 objects/s recovering 2024-09-13T14:47:23.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:47:23.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[108570]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:23.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:23 smithi042 ceph-mon[108570]: 10.18 scrub ok 2024-09-13T14:47:23.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:23.704 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:23.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:23 smithi159 ceph-mon[102519]: 10.6 scrub starts 2024-09-13T14:47:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:23 smithi159 ceph-mon[102519]: pgmap v683: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.1 KiB/s, 0 objects/s recovering 2024-09-13T14:47:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:47:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:23 smithi159 ceph-mon[102519]: osdmap e195: 8 total, 8 up, 7 in 2024-09-13T14:47:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:23 smithi159 ceph-mon[102519]: 10.18 scrub ok 2024-09-13T14:47:23.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:23.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:23.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:23.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:23.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:23.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:24.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:24.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:24.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 64445 64429 122.011 2.9375 0.00188224 0.0078374 2024-09-13T14:47:24.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:24.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:24.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:24.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:24.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:24.542 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:47:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:47:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:47:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:47:24.542 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:47:24.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:24.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:24.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:24.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:24.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:24.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: 10.7 scrub starts 2024-09-13T14:47:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: 10.6 scrub ok 2024-09-13T14:47:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: 10.7 scrub ok 2024-09-13T14:47:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:47:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:47:24.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: 10.7 scrub starts 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: 10.6 scrub ok 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: 10.7 scrub ok 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:47:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: 10.7 scrub starts 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: 10.6 scrub ok 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: 10.7 scrub ok 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: osdmap e196: 8 total, 8 up, 7 in 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:47:24.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:47:24.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:24.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:24.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:24.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:24.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:25.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:25.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 64862 64846 119.189 26.0625 0.00361963 0.00833885 2024-09-13T14:47:25.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:25.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:25.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:25.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:25.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:25.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:47:25.348 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:25.370 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:25.472 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:25.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:25.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:25.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:25.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:25.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:25.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:25 smithi159 ceph-mon[102519]: 10.9 scrub starts 2024-09-13T14:47:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:25 smithi159 ceph-mon[102519]: pgmap v686: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:25 smithi159 ceph-mon[102519]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:25 smithi159 ceph-mon[102519]: 10.9 scrub ok 2024-09-13T14:47:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:25 smithi159 ceph-mon[102519]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:25.753 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[113926]: 10.9 scrub starts 2024-09-13T14:47:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[113926]: pgmap v686: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[113926]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[113926]: 10.9 scrub ok 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[113926]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[108570]: 10.9 scrub starts 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[108570]: pgmap v686: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 3.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[108570]: osdmap e197: 8 total, 8 up, 7 in 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[108570]: 10.9 scrub ok 2024-09-13T14:47:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:25 smithi042 ceph-mon[108570]: osdmap e198: 8 total, 8 up, 7 in 2024-09-13T14:47:25.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:25.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:25.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:25.914 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:25.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:26.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:26.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 65070 65054 116.155 13 0.00313301 0.00832391 2024-09-13T14:47:26.105 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:26.127 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:26.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:26.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:26.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:26.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:26.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:26.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[113926]: 10.16 scrub starts 2024-09-13T14:47:26.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:47:26.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:47:26.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[113926]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:26.505 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[108570]: 10.16 scrub starts 2024-09-13T14:47:26.506 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:47:26.506 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:47:26.506 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:26 smithi042 ceph-mon[108570]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:26.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:26.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:26.625 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:26.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:26.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:26.691 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:26 smithi159 ceph-mon[102519]: 10.16 scrub starts 2024-09-13T14:47:26.692 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:47:26.692 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:47:26.692 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:26 smithi159 ceph-mon[102519]: osdmap e199: 8 total, 8 up, 7 in 2024-09-13T14:47:26.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:26.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:26.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:26.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:26.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:26.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:26.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:27.005 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:27.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:27.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 65492 65476 113.661 26.375 0.00192924 0.00868352 2024-09-13T14:47:27.142 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:27.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:27.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:27.290 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:27.391 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:27.413 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:27.515 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.640 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:27.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:27.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:27.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:27.789 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:27.891 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:27.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:27.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:27.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:28.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:28.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:28.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 65492 65476 110.589 0 - 0.00868352 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[108570]: pgmap v689: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[108570]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[113926]: pgmap v689: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:27 smithi042 ceph-mon[113926]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:28.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:28.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:28.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:28.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:27 smithi159 ceph-mon[102519]: pgmap v689: 109 pgs: 109 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail 2024-09-13T14:47:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:27 smithi159 ceph-mon[102519]: osdmap e200: 8 total, 8 up, 7 in 2024-09-13T14:47:28.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:28.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:28.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:28.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:28.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:28.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:28.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:28.660 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:28.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:28.785 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:28.886 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:28.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:28.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:29.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.044 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:29.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 65615 65599 107.881 3.84375 0.00890831 0.00926532 2024-09-13T14:47:29.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:29.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.284 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:29.321 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:29.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:29.464 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:29.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:29.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[108570]: pgmap v692: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:47:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[108570]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 2038/194937 objects degraded (1.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[113926]: pgmap v692: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:47:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[113926]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 2038/194937 objects degraded (1.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:29.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:29.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:29.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:29 smithi159 ceph-mon[102519]: pgmap v692: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:47:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:29 smithi159 ceph-mon[102519]: osdmap e201: 8 total, 8 up, 7 in 2024-09-13T14:47:29.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:29 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 2038/194937 objects degraded (1.045%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:29.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:29.855 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:29.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:29.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:29.957 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:29] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:47:29.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 67090 67074 107.478 92.1875 0.00406246 0.00929063 2024-09-13T14:47:30.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:30.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:30.248 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:30.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:30.386 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.522 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:30.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:30.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:30.690 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:30.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:30.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.835 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:30.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:30.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:30.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:30.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:30.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:31.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:31.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:47:31.097728+0000 min lat: 0.0013466 max lat: 2.54826 avg lat: 0.00933572 2024-09-13T14:47:31.097 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:47:31.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 68539 68523 107.055 90.5625 0.0103657 0.00933572 2024-09-13T14:47:31.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:31.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:31.205 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:31.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:31.360 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:31.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:31.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:31.398 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:47:31.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:31.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:31.533 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:31 smithi159 ceph-mon[102519]: pgmap v694: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:31.533 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:31 smithi042 ceph-mon[108570]: pgmap v694: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:31 smithi042 ceph-mon[113926]: pgmap v694: 109 pgs: 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 2038/194937 objects degraded (1.045%); 3640/194937 objects misplaced (1.867%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:47:31.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:31.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:31.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:31.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:31.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:31.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:31.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:31.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:32.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.033 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:119559 9.e 9:79d59a3d:::benchmark_data_smithi042_157727_object119558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:34.925997+0000", 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 236.487144782, 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99952705200000003, 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.034 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 119559 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.925997+0000", 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.925997+0000", 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.925996+0000", 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.926163+0000", 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016724700000000001 2024-09-13T14:47:32.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.926171+0000", 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0350000000000001e-06 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.926177+0000", 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3689999999999998e-06 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.926221+0000", 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4841e-05 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.926232+0000", 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0309000000000001e-05 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.911414+0000", 2024-09-13T14:47:32.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.985181847 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.913828+0000", 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024146089999999999 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.913936+0000", 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010825899999999999 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925349+0000", 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.011412528999999999 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925357+0000", 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2609999999999993e-06 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925427+0000", 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9643999999999999e-05 2024-09-13T14:47:32.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925481+0000", 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3884000000000001e-05 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925524+0000", 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3543e-05 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:119582 9.e 9:7457dd17:::benchmark_data_smithi042_157727_object119581:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:34.955594+0000", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 236.45754840999999, 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97744251500000001, 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 119582 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.042 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955594+0000", 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955594+0000", 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955592+0000", 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955702+0000", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011034700000000001 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955714+0000", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2055999999999999e-05 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955730+0000", 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5764e-05 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955774+0000", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3634999999999998e-05 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.955781+0000", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8250000000000005e-06 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.917716+0000", 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96193437699999995 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.917867+0000", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015170900000000001 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.918035+0000", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016725099999999999 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925394+0000", 2024-09-13T14:47:32.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073592099999999997 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925575+0000", 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000181127 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.932968+0000", 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073934379999999996 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933010+0000", 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1174000000000003e-05 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933036+0000", 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6545000000000001e-05 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:119585 9.e 9:7785ba02:::benchmark_data_smithi042_157727_object119584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:47:32.048 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:34.956073+0000", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 236.45706944099999, 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97710179399999997, 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 119585 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956073+0000", 2024-09-13T14:47:32.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956073+0000", 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956072+0000", 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956211+0000", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000138753 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956219+0000", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8780000000000004e-06 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956225+0000", 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9510000000000003e-06 2024-09-13T14:47:32.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956267+0000", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1597999999999997e-05 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956276+0000", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1360000000000003e-06 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.921697+0000", 2024-09-13T14:47:32.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96542099100000001 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.921865+0000", 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016758800000000001 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.922027+0000", 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000161996 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.932856+0000", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010829335000000001 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.932918+0000", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1781999999999999e-05 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933089+0000", 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017115000000000001 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933140+0000", 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0612999999999998e-05 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933174+0000", 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4804000000000001e-05 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:119586 9.e 9:7897324d:::benchmark_data_smithi042_157727_object119585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:34.956233+0000", 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 236.456909174, 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98478358099999996, 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 119586 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956233+0000", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956233+0000", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956232+0000", 2024-09-13T14:47:32.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956266+0000", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3927000000000002e-05 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956275+0000", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7919999999999998e-06 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956281+0000", 2024-09-13T14:47:32.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0050000000000001e-06 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956308+0000", 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6682999999999998e-05 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:34.956315+0000", 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9700000000000002e-06 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925598+0000", 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96928291499999997 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925740+0000", 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014187299999999999 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.925877+0000", 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000137959 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.933046+0000", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071683700000000003 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.940888+0000", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.007842267 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.940920+0000", 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1739999999999998e-05 2024-09-13T14:47:32.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.940964+0000", 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3835000000000003e-05 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:35.941016+0000", 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2805e-05 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64096 10.10 10:0ab9fd2a:::benchmark_data_smithi042_191918_object64095:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.448184+0000", 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.96495803, 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.984902999, 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64096 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.063 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448184+0000", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448184+0000", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448182+0000", 2024-09-13T14:47:32.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448249+0000", 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7335999999999994e-05 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448264+0000", 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4139e-05 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448271+0000", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1919999999999999e-06 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448322+0000", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1051000000000003e-05 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.448332+0000", 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8600000000000005e-06 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365379+0000", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91704745600000004 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365391+0000", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1831000000000001e-05 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.429002+0000", 2024-09-13T14:47:32.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0636112019999999 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.429121+0000", 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000118807 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.429229+0000", 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107947 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.068 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.432162+0000", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0029335849999999998 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.432202+0000", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9146000000000001e-05 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.432265+0000", 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3768000000000003e-05 2024-09-13T14:47:32.069 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.432307+0000", 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2048999999999997e-05 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.433087+0000", 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00077962500000000004 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.070 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64109 10.10 10:0ff677dd:::benchmark_data_smithi042_191918_object64108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.457450+0000", 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.9556921729999992, 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9777625589999999, 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64109 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.071 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457450+0000", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457450+0000", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457448+0000", 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457485+0000", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6754000000000001e-05 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457497+0000", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2286000000000001e-05 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457504+0000", 2024-09-13T14:47:32.073 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3629999999999999e-06 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457583+0000", 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9859999999999998e-05 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.457594+0000", 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0998e-05 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.074 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365423+0000", 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90782886900000004 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365431+0000", 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6729999999999992e-06 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.431074+0000", 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0656431340000001 2024-09-13T14:47:32.075 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.431182+0000", 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010801799999999999 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.431283+0000", 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010114 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.076 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434271+0000", 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002987806 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434324+0000", 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2583999999999997e-05 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434375+0000", 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1523999999999997e-05 2024-09-13T14:47:32.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434452+0000", 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6532999999999994e-05 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.435212+0000", 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00076071699999999995 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.078 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64114 10.10 10:0a989e6c:::benchmark_data_smithi042_191918_object64113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.458079+0000", 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.9550626290000004, 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.979283366, 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64114 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.079 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458079+0000", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458079+0000", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458079+0000", 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458110+0000", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1072000000000003e-05 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458119+0000", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8120000000000002e-06 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458128+0000", 2024-09-13T14:47:32.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1070000000000005e-06 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458174+0000", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6427999999999997e-05 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.458183+0000", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2690000000000005e-06 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365455+0000", 2024-09-13T14:47:32.082 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90727170599999996 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365463+0000", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.433115+0000", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.067652582 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.433239+0000", 2024-09-13T14:47:32.083 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012343 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.433377+0000", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000137887 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.436224+0000", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002847572 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.436879+0000", 2024-09-13T14:47:32.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000654684 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.436912+0000", 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3479000000000003e-05 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.436947+0000", 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4594000000000001e-05 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.085 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437363+0000", 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041597100000000001 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64152 10.10 10:0d876128:::benchmark_data_smithi042_191918_object64151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.482033+0000", 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.9311090699999998, 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9590801070000001, 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.086 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64152 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482033+0000", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482033+0000", 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.087 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482032+0000", 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482234+0000", 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020251500000000001 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482277+0000", 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2221000000000001e-05 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482284+0000", 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0129999999999996e-06 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482337+0000", 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3470000000000001e-05 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.482350+0000", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3149000000000001e-05 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365486+0000", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.88313579399999997 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365493+0000", 2024-09-13T14:47:32.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2590000000000003e-06 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.435244+0000", 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.069750505 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.435347+0000", 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010270800000000001 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.435449+0000", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000102898 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437402+0000", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001952921 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437411+0000", 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5140000000000001e-06 2024-09-13T14:47:32.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.439882+0000", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024711020000000002 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.439942+0000", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.978e-05 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.441113+0000", 2024-09-13T14:47:32.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011713520000000001 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64207 10.10 10:0e8f2d81:::benchmark_data_smithi042_191918_object64206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.523584+0000", 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.8895584139999997, 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9232562560000002, 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.094 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64207 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.523584+0000", 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.523584+0000", 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.523583+0000", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.523647+0000", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3911999999999994e-05 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.552144+0000", 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028496807999999998 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.096 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.552154+0000", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0597e-05 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.552213+0000", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8511999999999998e-05 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.552224+0000", 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1698999999999999e-05 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.097 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365525+0000", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.81330031800000002 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365533+0000", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7419999999999997e-06 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437434+0000", 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0719012370000001 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.098 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437594+0000", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016047699999999999 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.437686+0000", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2072999999999998e-05 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.441176+0000", 2024-09-13T14:47:32.099 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034901580000000001 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.441242+0000", 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5699999999999998e-05 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.445909+0000", 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046665789999999997 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.100 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.445946+0000", 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7734999999999997e-05 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.446840+0000", 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00089345799999999997 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64239 10.10 10:0eebd97f:::benchmark_data_smithi042_191918_object64238:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.586401+0000", 2024-09-13T14:47:32.101 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.8267409089999997, 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8611178339999999, 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64239 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586401+0000", 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.102 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586401+0000", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586401+0000", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586429+0000", 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8211000000000001e-05 2024-09-13T14:47:32.103 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586436+0000", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7270000000000007e-06 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586442+0000", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8320000000000002e-06 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586492+0000", 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0031000000000003e-05 2024-09-13T14:47:32.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.586503+0000", 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1168999999999999e-05 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365556+0000", 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77905286200000001 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365562+0000", 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0689999999999998e-06 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.438713+0000", 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.073150695 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.438862+0000", 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014849899999999999 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.106 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.439021+0000", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015930999999999999 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.441229+0000", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022083509999999999 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.441251+0000", 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1257999999999999e-05 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.107 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.446908+0000", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056573889999999996 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.446962+0000", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3600000000000002e-05 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.447519+0000", 2024-09-13T14:47:32.108 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00055748799999999997 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64272 10.10 10:0af4c38b:::benchmark_data_smithi042_191918_object64271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.610871+0000", 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.8022715070000004, 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8372639560000001, 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.109 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64272 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610871+0000", 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610871+0000", 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.110 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610870+0000", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610896+0000", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5602999999999999e-05 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610903+0000", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7810000000000005e-06 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.111 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.610909+0000", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6670000000000001e-06 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.611058+0000", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000149281 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.611067+0000", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8990000000000007e-06 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.112 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365599+0000", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.75453225400000001 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365609+0000", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1970000000000002e-06 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.444694+0000", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0790856470000001 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.113 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.444801+0000", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106748 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.444905+0000", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103806 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.447565+0000", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026603640000000001 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.114 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.447615+0000", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9391000000000002e-05 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.447625+0000", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0121e-05 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.447676+0000", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1557999999999997e-05 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.448135+0000", 2024-09-13T14:47:32.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00045821099999999998 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64279 10.10 10:0e08e071:::benchmark_data_smithi042_191918_object64278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.614024+0000", 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7991182109999997, 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8384259460000001, 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.116 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64279 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614024+0000", 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614024+0000", 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.117 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614023+0000", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614062+0000", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8693999999999999e-05 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614077+0000", 2024-09-13T14:47:32.118 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5617e-05 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614084+0000", 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5869999999999998e-06 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614126+0000", 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2203999999999998e-05 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.614136+0000", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0424999999999999e-05 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365631+0000", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.75149507299999996 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365639+0000", 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1840000000000002e-06 2024-09-13T14:47:32.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.448164+0000", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0825249290000001 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.448269+0000", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000105688 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.448374+0000", 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000104516 2024-09-13T14:47:32.121 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.451470+0000", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030966079999999998 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.451481+0000", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0550000000000001e-05 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.451543+0000", 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2364000000000001e-05 2024-09-13T14:47:32.122 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.451602+0000", 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8406999999999998e-05 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452450+0000", 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00084810199999999999 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.123 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64286 10.10 10:0ee76fa3:::benchmark_data_smithi042_191918_object64285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.620914+0000", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7922278410000008, 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.835550231, 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64286 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.124 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620914+0000", 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620914+0000", 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620913+0000", 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.125 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620961+0000", 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8458000000000003e-05 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620977+0000", 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5486e-05 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.126 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.620985+0000", 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0420000000000003e-06 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.621049+0000", 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4623999999999997e-05 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.621062+0000", 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2223e-05 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.127 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365674+0000", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.74461248800000002 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365680+0000", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9020000000000004e-06 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.449246+0000", 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.08356648 2024-09-13T14:47:32.128 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.449404+0000", 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015713500000000001 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.449559+0000", 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015509599999999999 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.129 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452501+0000", 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002941876 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.453877+0000", 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013761979999999999 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.455228+0000", 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001351388 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.130 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.455282+0000", 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4187000000000001e-05 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.456464+0000", 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001182118 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.131 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64307 10.10 10:08e0f1be:::benchmark_data_smithi042_191918_object64306:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.634733+0000", 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7784087779999993, 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8226378169999999, 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.132 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64307 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634733+0000", 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634733+0000", 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.133 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634732+0000", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634796+0000", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3754e-05 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634808+0000", 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2619999999999999e-05 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.134 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634816+0000", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3379999999999999e-06 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634950+0000", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000134318 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.634961+0000", 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1116e-05 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.135 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365700+0000", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73073923299999999 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365707+0000", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1180000000000003e-06 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.450408+0000", 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.084700979 2024-09-13T14:47:32.136 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.450562+0000", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015345499999999999 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.450686+0000", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012392999999999999 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452548+0000", 2024-09-13T14:47:32.137 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001862378 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.456483+0000", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039350110000000004 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.456550+0000", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7138000000000007e-05 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.456607+0000", 2024-09-13T14:47:32.138 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6873000000000002e-05 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.457371+0000", 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00076394600000000005 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64330 10.10 10:0e8f4621:::benchmark_data_smithi042_191918_object64329:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.653811+0000", 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7593307399999993, 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8043665390000001, 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.139 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64330 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653811+0000", 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.140 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653811+0000", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653810+0000", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653863+0000", 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2442999999999999e-05 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.141 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653872+0000", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2550000000000003e-06 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653877+0000", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3600000000000004e-06 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653928+0000", 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1245999999999997e-05 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.142 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.143 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 69790 69774 106.351 78.1875 0.0136409 0.00939737 2024-09-13T14:47:32.143 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.653939+0000", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0698e-05 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365729+0000", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.711790276 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365733+0000", 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0430000000000001e-06 2024-09-13T14:47:32.144 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452588+0000", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.086854948 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452724+0000", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013521200000000001 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.452878+0000", 2024-09-13T14:47:32.145 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000154007 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.457425+0000", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045468720000000004 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.457498+0000", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3186e-05 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.457508+0000", 2024-09-13T14:47:32.146 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0351e-05 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.457569+0000", 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.05e-05 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458178+0000", 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00060932499999999997 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.147 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64346 10.10 10:08a4bcaf:::benchmark_data_smithi042_191918_object64345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.667208+0000", 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7459337749999992, 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.791693091, 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64346 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.148 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667208+0000", 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667208+0000", 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.149 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667207+0000", 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667281+0000", 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3234999999999999e-05 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667291+0000", 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0329999999999999e-05 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.150 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667297+0000", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3040000000000001e-06 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667342+0000", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4280000000000003e-05 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.667350+0000", 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7690000000000002e-06 2024-09-13T14:47:32.151 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365752+0000", 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.69840210400000002 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365757+0000", 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0860000000000004e-06 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.152 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.453923+0000", 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.088166601 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.454078+0000", 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015522800000000001 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.454260+0000", 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018148700000000001 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.153 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458193+0000", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039331110000000004 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458237+0000", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3906999999999998e-05 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458282+0000", 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4650000000000001e-05 2024-09-13T14:47:32.154 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458336+0000", 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4397000000000001e-05 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458901+0000", 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00056546099999999996 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.155 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64350 10.10 10:0deaa3df:::benchmark_data_smithi042_191918_object64349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.671030+0000", 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.7421122239999995, 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.793744027, 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64350 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.156 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671030+0000", 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671030+0000", 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.157 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671029+0000", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671098+0000", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9719000000000001e-05 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671111+0000", 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2303999999999999e-05 2024-09-13T14:47:32.158 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671117+0000", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6579999999999996e-06 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671165+0000", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8096000000000002e-05 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.671175+0000", 2024-09-13T14:47:32.159 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4499999999999993e-06 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365773+0000", 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.69459839199999995 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365777+0000", 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6739999999999999e-06 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.160 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.458942+0000", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0931654040000001 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.459101+0000", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000158534 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.459243+0000", 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014215000000000001 2024-09-13T14:47:32.161 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.461628+0000", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023845889999999999 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.462467+0000", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00083908300000000002 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.463552+0000", 2024-09-13T14:47:32.162 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0010848839999999999 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.463607+0000", 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5248999999999999e-05 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.464774+0000", 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011669829999999999 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.163 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64367 10.10 10:08b7e77a:::benchmark_data_smithi042_191918_object64366:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:22.982278+0000", 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.4308637350000009, 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.487172591, 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64367 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.164 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982278+0000", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982278+0000", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982278+0000", 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.165 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982329+0000", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1719999999999999e-05 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982336+0000", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0369999999999997e-06 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982340+0000", 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0799999999999999e-06 2024-09-13T14:47:32.166 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982375+0000", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4820999999999997e-05 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:22.982381+0000", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.48e-06 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365793+0000", 2024-09-13T14:47:32.167 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.383412578 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365797+0000", 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7069999999999998e-06 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.460381+0000", 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.094583525 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.168 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.460540+0000", 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015923599999999999 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.460704+0000", 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000164406 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.463485+0000", 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027809509999999998 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.169 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.466026+0000", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0025413889999999998 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.468863+0000", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002836295 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.468920+0000", 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6713999999999999e-05 2024-09-13T14:47:32.170 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.469451+0000", 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00053143000000000005 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64376 10.10 10:0a9cd48d:::benchmark_data_smithi042_191918_object64375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:23.039910+0000", 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.3732321120000002, 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4301207419999999, 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.171 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64376 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039910+0000", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039910+0000", 2024-09-13T14:47:32.172 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039908+0000", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039956+0000", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7451999999999999e-05 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039968+0000", 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2193e-05 2024-09-13T14:47:32.173 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.039973+0000", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3730000000000001e-06 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.040019+0000", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6011999999999999e-05 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.040026+0000", 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5289999999999997e-06 2024-09-13T14:47:32.174 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365819+0000", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.32579311799999999 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365823+0000", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0709999999999996e-06 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.461659+0000", 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0958359209999999 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.175 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.461778+0000", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011952500000000001 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.461875+0000", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6197000000000001e-05 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.463505+0000", 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0016307310000000001 2024-09-13T14:47:32.176 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.468814+0000", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053086219999999998 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.469496+0000", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00068234800000000005 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.469542+0000", 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5787000000000001e-05 2024-09-13T14:47:32.177 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.470031+0000", 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048860799999999997 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64445 10.10 10:0b9591ff:::benchmark_data_smithi042_191918_object64444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e195)", 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:23.227129+0000", 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 8.1860135760000006, 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.243657225, 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:32.178 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64445 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227129+0000", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227129+0000", 2024-09-13T14:47:32.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227128+0000", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227183+0000", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5856999999999997e-05 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227196+0000", 2024-09-13T14:47:32.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.287e-05 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227203+0000", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9609999999999999e-06 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227254+0000", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1181999999999999e-05 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.227262+0000", 2024-09-13T14:47:32.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9920000000000007e-06 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365836+0000", 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13857381299999999 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:23.365840+0000", 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8800000000000001e-06 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:32.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.462494+0000", 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0966540359999999 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.462646+0000", 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015201799999999999 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.462793+0000", 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014713199999999999 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.466061+0000", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00326809 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.468823+0000", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027614739999999999 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.470092+0000", 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0012695300000000001 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.470144+0000", 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1796999999999999e-05 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.470786+0000", 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00064148600000000003 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:32.185 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:47:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:32.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:32.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:32.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:32.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:32.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:32.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:32.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[113926]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:32.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:32 smithi042 ceph-mon[108570]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:32.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:32.744 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:32.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3898480451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:47:32.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-mon[102519]: osdmap e202: 8 total, 8 up, 7 in 2024-09-13T14:47:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=cleanup t=2024-09-13T14:47:32.298964795Z level=info msg="Completed cleanup jobs" duration=5.422507ms 2024-09-13T14:47:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugins.update.checker t=2024-09-13T14:47:32.472841017Z level=info msg="Update check succeeded" duration=69.754554ms 2024-09-13T14:47:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:47:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana.update.checker t=2024-09-13T14:47:32.47320919Z level=info msg="Update check succeeded" duration=70.148808ms 2024-09-13T14:47:32.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:32.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:32.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:32.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:33.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:33.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:33.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 71231 71215 105.963 90.0625 0.00700981 0.00943233 2024-09-13T14:47:33.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:33.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:33.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:33.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:33.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:33.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-09-13T14:47:33.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:33.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:47:33.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:33.603 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:33.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:33.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:33.716 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:33 smithi042 ceph-mon[113926]: pgmap v696: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s; 2038/203913 objects degraded (0.999%); 2776/203913 objects misplaced (1.361%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:47:33.719 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:33 smithi042 ceph-mon[108570]: pgmap v696: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s; 2038/203913 objects degraded (0.999%); 2776/203913 objects misplaced (1.361%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:47:33.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:33 smithi159 ceph-mon[102519]: pgmap v696: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 630 op/s; 2038/203913 objects degraded (0.999%); 2776/203913 objects misplaced (1.361%); 13 MiB/s, 207 objects/s recovering 2024-09-13T14:47:33.751 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:33.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:33.898 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:33.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:34.000 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:34.035 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:47:34.036 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:34.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:34.061 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:47:34.061 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:34.064 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:34.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:47:34.091 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:34.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 72629 72613 105.531 87.375 0.00788646 0.00946941 2024-09-13T14:47:34.117 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:47:34.117 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:34.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:34.156 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:47:34.157 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:34.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:34.190 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:47:34.191 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:34.233 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:47:34.233 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:34.236 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:34.260 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:47:34.260 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:34.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:34.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:34.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:34.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:34.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:34.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1324542672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:34.557 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.557 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.557 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1324542672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:34.557 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.557 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:34.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:34.643 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1324542672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:34.643 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.644 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:34.700 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:34.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:34.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:47:34.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:34.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:34.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:34.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:34.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 73947 73931 105.004 82.375 0.00712894 0.00951851 2024-09-13T14:47:35.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:35.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:35.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:35.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:35.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:35.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:35.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:35.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:35.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:35.656 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:35 smithi159 ceph-mon[102519]: pgmap v697: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 6180/205647 objects degraded (3.005%); 4847/205647 objects misplaced (2.357%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:47:35.656 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:35.656 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:35.656 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:35.656 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6180/205647 objects degraded (3.005%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:35.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.728 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:35.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[108570]: pgmap v697: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 6180/205647 objects degraded (3.005%); 4847/205647 objects misplaced (2.357%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6180/205647 objects degraded (3.005%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:35.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[113926]: pgmap v697: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 33 MiB/s wr, 532 op/s; 6180/205647 objects degraded (3.005%); 4847/205647 objects misplaced (2.357%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:47:35.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:47:35.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:47:35.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:35.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6180/205647 objects degraded (3.005%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:35.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:35.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:35.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:36.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 75339 75323 104.604 87 0.0165857 0.00955456 2024-09-13T14:47:36.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:36.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:36.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:36.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:36.380 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:36.380 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:36.407 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:36.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:36.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:36.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:36.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:36.676 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:36.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:36.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:36.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:36.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:36.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:36.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:37.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:37.085 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:37.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 76705 76689 104.186 85.375 0.00345376 0.00958384 2024-09-13T14:47:37.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:37.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:37.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:37.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:47:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:37.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:37.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:37.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:37.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:37.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:37.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:37.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:37.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:37.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:37.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:37.887 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:37.921 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:37.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:38.023 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:37 smithi042 ceph-mon[108570]: pgmap v698: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 999 op/s; 6180/218922 objects degraded (2.823%); 4847/218922 objects misplaced (2.214%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:47:38.032 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:37 smithi042 ceph-mon[113926]: pgmap v698: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 999 op/s; 6180/218922 objects degraded (2.823%); 4847/218922 objects misplaced (2.214%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:47:38.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:38.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:38.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 77940 77924 103.611 77.1875 0.00230243 0.00964628 2024-09-13T14:47:38.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:38.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:38.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:37 smithi159 ceph-mon[102519]: pgmap v698: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 62 MiB/s wr, 999 op/s; 6180/218922 objects degraded (2.823%); 4847/218922 objects misplaced (2.214%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T14:47:38.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:38.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:38.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:38.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:38.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-09-13T14:47:38.490 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:38.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:38.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:38.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:38.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.730 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:38.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.832 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:38.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:38.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:38.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:38.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:38.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:38.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:39.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 78741 78725 102.495 50.0625 0.00299012 0.00972545 2024-09-13T14:47:39.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:47:39.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:39.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:39.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:47:39.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:39.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.241 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:39.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:39.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:39.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:39.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:39.487 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:39.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:39.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:39.582 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:39.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:39.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:39.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:39.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:39.814 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:39.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:39.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:39.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:39.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:39.990 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:39.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:39 smithi159 ceph-mon[102519]: pgmap v699: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:47:40.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:40.078 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:39 smithi042 ceph-mon[108570]: pgmap v699: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:47:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:39 smithi042 ceph-mon[113926]: pgmap v699: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 65 MiB/s wr, 1.04k op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 8.5 MiB/s, 135 objects/s recovering 2024-09-13T14:47:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:39] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-13T14:47:40.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 79853 79837 101.822 69.5 0.00528847 0.00980673 2024-09-13T14:47:40.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:40.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:40.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:40.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:40.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:40.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:40.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:40.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:40.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.845 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:40.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:40.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:40.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:41.065 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:40 smithi042 ceph-mon[113926]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:41.065 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6180/226104 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:41.066 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:40 smithi042 ceph-mon[108570]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:41.066 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6180/226104 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:41.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:41.098 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 80493 80477 100.585 40 0.0056737 0.00993599 2024-09-13T14:47:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:41.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:41.164 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:40 smithi159 ceph-mon[102519]: osdmap e203: 8 total, 8 up, 7 in 2024-09-13T14:47:41.164 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6180/226104 objects degraded (2.733%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:47:41.221 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:41.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:41.360 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:41.392 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:41.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:41.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:41.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:41.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:41.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:41.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:41.785 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:41.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:41.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:41.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:41.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:41 smithi159 ceph-mon[102519]: pgmap v701: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:47:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:41 smithi159 ceph-mon[102519]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:42.047 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:42.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:42.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:41 smithi042 ceph-mon[113926]: pgmap v701: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:47:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:41 smithi042 ceph-mon[113926]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:41 smithi042 ceph-mon[108570]: pgmap v701: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 938 op/s; 6180/226104 objects degraded (2.733%); 4364/226104 objects misplaced (1.930%); 9.5 MiB/s, 151 objects/s recovering 2024-09-13T14:47:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:41 smithi042 ceph-mon[108570]: osdmap e204: 8 total, 8 up, 7 in 2024-09-13T14:47:42.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 81850 81834 100.276 84.8125 0.0113328 0.00996749 2024-09-13T14:47:42.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:42.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:42.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:42.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.464 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:42.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:42.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.740 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.842 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:42.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:42.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:42.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:43.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:43.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 82839 82823 99.5362 61.8125 0.00311727 0.0100309 2024-09-13T14:47:43.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:43.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:43.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:43.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:43.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:43.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.540 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:43.540 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:43.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:43.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:43.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:43.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:43.822 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:47:43.851 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:43.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:43.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:43.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:44.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[113926]: pgmap v703: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6180/240531 objects degraded (2.569%); 4109/240531 objects misplaced (1.708%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:47:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1894414651' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[108570]: pgmap v703: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6180/240531 objects degraded (2.569%); 4109/240531 objects misplaced (1.708%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:47:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1894414651' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 83957 83941 98.9764 69.875 0.00683177 0.0100981 2024-09-13T14:47:44.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:44.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:44.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:43 smithi159 ceph-mon[102519]: pgmap v703: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 6180/240531 objects degraded (2.569%); 4109/240531 objects misplaced (1.708%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:47:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1894414651' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:47:44.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:47:44.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:44.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:44.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:44.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:44.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:44.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:44.728 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:44.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:44.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:44.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:44.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:44.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:45.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:45.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:45.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:44 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:44 smithi159 ceph-mon[102519]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[108570]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:47:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:44 smithi042 ceph-mon[113926]: osdmap e205: 8 total, 8 up, 7 in 2024-09-13T14:47:45.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 85185 85169 98.5647 76.75 0.0185619 0.0101407 2024-09-13T14:47:45.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.196 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:45.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:45.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.336 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:45.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:45.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:45.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:45.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:45.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:45.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:45.751 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:45.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:45.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:45.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:45.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:45.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:46.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:46.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:46.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:45 smithi042 ceph-mon[113926]: pgmap v705: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 4142/241788 objects degraded (1.713%); 5693/241788 objects misplaced (2.355%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:47:46.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/241788 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:46.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:45 smithi042 ceph-mon[108570]: pgmap v705: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 4142/241788 objects degraded (1.713%); 5693/241788 objects misplaced (2.355%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:47:46.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/241788 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:46.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 86477 86461 98.2407 80.75 0.00973759 0.010174 2024-09-13T14:47:46.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:46.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:46.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:45 smithi159 ceph-mon[102519]: pgmap v705: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 4142/241788 objects degraded (1.713%); 5693/241788 objects misplaced (2.355%); 5.0 MiB/s, 80 objects/s recovering 2024-09-13T14:47:46.301 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/241788 objects degraded (1.713%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:46.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:46.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:46.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:46.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:46.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:46.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:46.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:46.755 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:46.858 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:46.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:46.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:46.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:47.008 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:47.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:47.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 87816 87800 97.9806 83.6875 0.00595828 0.010201 2024-09-13T14:47:47.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:47.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:47.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:47.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:47.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:47.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:47.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:47.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:47.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:47.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:47.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:47.832 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:47.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:47.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:47.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:48.010 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:48.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:48.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:47 smithi042 ceph-mon[108570]: pgmap v706: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 4142/253551 objects degraded (1.634%); 5693/253551 objects misplaced (2.245%); 4.6 MiB/s, 74 objects/s recovering 2024-09-13T14:47:48.084 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:47 smithi042 ceph-mon[113926]: pgmap v706: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 4142/253551 objects degraded (1.634%); 5693/253551 objects misplaced (2.245%); 4.6 MiB/s, 74 objects/s recovering 2024-09-13T14:47:48.099 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 88947 88931 97.5017 70.6875 0.0203615 0.0102513 2024-09-13T14:47:48.102 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:47 smithi159 ceph-mon[102519]: pgmap v706: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 88 MiB/s wr, 1.41k op/s; 4142/253551 objects degraded (1.634%); 5693/253551 objects misplaced (2.245%); 4.6 MiB/s, 74 objects/s recovering 2024-09-13T14:47:48.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.243 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:48.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:48.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:48.339 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:47:48.339 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:47:48.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:48.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:47:48.528 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:48.529 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:48.530 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:48.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:48.557 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:47:48.558 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:48.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:48.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:48.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:48.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:48.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:48.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:48.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:48.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/848120638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:49.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/848120638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:49.036 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-09-13T14:47:49.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:49.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 89973 89957 96.9251 64.125 0.00764458 0.0103125 2024-09-13T14:47:49.128 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:49.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:49.230 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:49.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/848120638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:49.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:49.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:49.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:49.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:49.495 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:49.518 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:49.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:49.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:47:49.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.669 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:49.771 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:49.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:49.906 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:49.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:49.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:50.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:50.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.083 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:50.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:49 smithi042 ceph-mon[113926]: pgmap v707: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:47:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:49 smithi042 ceph-mon[108570]: pgmap v707: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:47:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:49] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.51.0" 2024-09-13T14:47:50.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 90891 90875 96.2547 57.375 0.00992231 0.0103843 2024-09-13T14:47:50.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:50.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:49 smithi159 ceph-mon[102519]: pgmap v707: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 87 MiB/s wr, 1.40k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:47:50.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.350 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:50.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:50.452 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:50.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:50.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:50.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:50.621 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:50.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:50.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:47:50.722 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:50.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:50.880 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:50.901 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/259704 objects degraded (1.595%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:50.901 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:50.911 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:50.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:51.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.045 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/259704 objects degraded (1.595%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:51.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/259704 objects degraded (1.595%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:47:51.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:47:51.100222+0000 min lat: 0.0013466 max lat: 2.54826 avg lat: 0.0104332 2024-09-13T14:47:51.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:47:51.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 91996 91980 95.8014 69.0625 0.0147941 0.0104332 2024-09-13T14:47:51.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:51.185 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:51.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:51.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:51.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:51.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:51.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:51.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:51.727 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:51.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:51.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:51.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:51.992 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:51.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:52.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:51 smithi042 ceph-mon[108570]: pgmap v708: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:51 smithi042 ceph-mon[113926]: pgmap v708: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:52.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 93155 93139 95.4183 72.4375 0.00623924 0.0104751 2024-09-13T14:47:52.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.148 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:52.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:51 smithi159 ceph-mon[102519]: pgmap v708: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 67 MiB/s wr, 1.06k op/s; 4142/259704 objects degraded (1.595%); 5217/259704 objects misplaced (2.009%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:47:52.249 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:52.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:52.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:52.432 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:52.445 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:52.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:52.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:52.679 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:52.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:52.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:52.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:52.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:52.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:53.010 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:53.086 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94405 10.d 10:b9766f91:::benchmark_data_smithi042_191918_object94404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.075028+0000", 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0088156399999999996, 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0088559669999999993, 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.087 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94405 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075028+0000", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075028+0000", 2024-09-13T14:47:53.088 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075027+0000", 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075073+0000", 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6727000000000002e-05 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075084+0000", 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0672e-05 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075095+0000", 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0458999999999999e-05 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:53.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075148+0000", 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3599e-05 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075277+0000", 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000129016 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.075408+0000", 2024-09-13T14:47:53.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013050200000000001 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94406 10.d 10:b2f1de3f:::benchmark_data_smithi042_191918_object94405:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.076735+0000", 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0071089389999999999, 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0072533060000000002, 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.092 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94406 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076735+0000", 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076735+0000", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076734+0000", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076769+0000", 2024-09-13T14:47:53.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5318999999999998e-05 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076778+0000", 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2539999999999998e-06 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.076784+0000", 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3180000000000003e-06 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.095 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078650+0000", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00186535 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078754+0000", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010450600000000001 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078880+0000", 2024-09-13T14:47:53.096 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012536000000000001 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94410 10.b 10:d8b69203:::benchmark_data_smithi042_191918_object94409:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.078011+0000", 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0058327300000000004, 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0060575919999999997, 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.097 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94410 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078011+0000", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078011+0000", 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.098 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078010+0000", 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078067+0000", 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7534000000000002e-05 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.099 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078082+0000", 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4744e-05 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078089+0000", 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4389999999999997e-06 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078603+0000", 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00051402200000000005 2024-09-13T14:47:53.100 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078725+0000", 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012236300000000001 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.078857+0000", 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013184700000000001 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:47:53.101 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082505+0000", 2024-09-13T14:47:53.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 94433 94417 95.1675 79.875 0.00688111 0.010503 2024-09-13T14:47:53.102 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036484360000000001 2024-09-13T14:47:53.102 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.102 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94411 10.f 10:f025ecdc:::benchmark_data_smithi042_191918_object94410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.081705+0000", 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0021384220000000001, 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024823039999999998, 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.103 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94411 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081705+0000", 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081705+0000", 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.104 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081704+0000", 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081734+0000", 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0762999999999997e-05 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.105 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081753+0000", 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8655e-05 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081762+0000", 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7339999999999997e-06 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081806+0000", 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4369999999999997e-05 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.106 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.081924+0000", 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000118263 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082046+0000", 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121311 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.107 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94412 10.f 10.eb34cabf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.082355+0000", 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0014888340000000001, 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0019354960000000001, 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94412 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.108 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082355+0000", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082355+0000", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082354+0000", 2024-09-13T14:47:53.109 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082425+0000", 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0679999999999994e-05 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082436+0000", 2024-09-13T14:47:53.110 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1181e-05 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.082443+0000", 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.238e-06 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45032.0:94405 10.d e205/184)", 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.083859+0000", 2024-09-13T14:47:53.111 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9999852, 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000485095, 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083859+0000", 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.112 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083859+0000", 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083858+0000", 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083863+0000", 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2920000000000003e-06 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.113 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083876+0000", 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3422e-05 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083883+0000", 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4130000000000003e-06 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.114 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45032.0:94406 10.d e205/184)", 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.083889+0000", 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9999547, 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050423199999999997, 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083889+0000", 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.115 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083889+0000", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083889+0000", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083891+0000", 2024-09-13T14:47:53.116 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.069e-06 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083896+0000", 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9520000000000002e-06 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.083900+0000", 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7060000000000002e-06 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.117 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:94417 10.f 10.1be6e37f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e205)", 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:53.084244+0000", 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9995999, 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00019098800000000001, 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:47:53.118 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 94417 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084244+0000", 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084244+0000", 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.119 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084243+0000", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084279+0000", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6254999999999998e-05 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084290+0000", 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1071e-05 2024-09-13T14:47:53.120 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:53.084297+0000", 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0330000000000003e-06 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 8 2024-09-13T14:47:53.121 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:47:53.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:53.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.220 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:53.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:53.303 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:53.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:53.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:53.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:53.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:53.582 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:53.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:53.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:53.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:53.826 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:53.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:53.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:53.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:47:53.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:54.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:54.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:53 smithi042 ceph-mon[113926]: pgmap v709: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 4142/274890 objects degraded (1.507%); 5217/274890 objects misplaced (1.898%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:53 smithi042 ceph-mon[108570]: pgmap v709: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 4142/274890 objects degraded (1.507%); 5217/274890 objects misplaced (1.898%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:54.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 95715 95699 94.9286 80.125 0.00730489 0.0105296 2024-09-13T14:47:54.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:54.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:53 smithi159 ceph-mon[102519]: pgmap v709: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 72 MiB/s wr, 1.14k op/s; 4142/274890 objects degraded (1.507%); 5217/274890 objects misplaced (1.898%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:54.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:54.264 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:54.312 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:54.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:47:54.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:54.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:54.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:54.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:54.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:54.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:54.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:54.751 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:54.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:54.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:54.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:54.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:54.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:55.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 96836 96820 94.54 70.0625 0.00286834 0.0105548 2024-09-13T14:47:55.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:55.144 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:55.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:55.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:55.281 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:55.383 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:55.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:55.528 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:55.567 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:55.669 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:55.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:55.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:55.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:55.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:55.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:55.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:55.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:55.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:56.082 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:56.095 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:55 smithi042 ceph-mon[113926]: pgmap v710: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/276459 objects degraded (1.498%); 4723/276459 objects misplaced (1.708%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:56.095 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/276459 objects degraded (1.498%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:56.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:55 smithi042 ceph-mon[108570]: pgmap v710: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/276459 objects degraded (1.498%); 4723/276459 objects misplaced (1.708%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:56.095 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/276459 objects degraded (1.498%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:56.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 97957 97941 94.1633 70.0625 0.0468334 0.0106123 2024-09-13T14:47:56.115 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:56.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:56.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:55 smithi159 ceph-mon[102519]: pgmap v710: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.6 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/276459 objects degraded (1.498%); 4723/276459 objects misplaced (1.708%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:47:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/276459 objects degraded (1.498%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:47:56.290 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:56.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:56.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:56.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:56.479 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:56.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:56.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:56.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:56.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:56.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:56.762 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:56.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:56.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:56.879 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:47:56.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:56.943 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:56.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:56.962 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:56 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1576452988' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:56.962 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:56 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:56.962 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:56 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1576452988' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:56.963 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:56 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:57.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.089 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:57.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 99080 99064 93.8 70.1875 0.0152179 0.0106563 2024-09-13T14:47:57.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:57.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:56 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1576452988' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:56 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:47:57.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.362 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:57.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-09-13T14:47:57.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:47:57.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:47:57.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:57.660 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:57.762 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.813 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:57.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:57.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:57.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:57.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:57 smithi159 ceph-mon[102519]: pgmap v711: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s; 4142/288135 objects degraded (1.438%); 4723/288135 objects misplaced (1.639%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:47:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:57 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:57 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:57 smithi159 ceph-mon[102519]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:58.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:58.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[108570]: pgmap v711: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s; 4142/288135 objects degraded (1.438%); 4723/288135 objects misplaced (1.639%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:47:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[108570]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[113926]: pgmap v711: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.9 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 80 MiB/s wr, 1.29k op/s; 4142/288135 objects degraded (1.438%); 4723/288135 objects misplaced (1.639%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:47:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:47:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:47:58.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:57 smithi042 ceph-mon[113926]: osdmap e206: 8 total, 8 up, 7 in 2024-09-13T14:47:58.100 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 100333 100317 93.5687 78.3125 0.0129726 0.0106818 2024-09-13T14:47:58.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:58.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:58.346 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:58.355 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:47:58.356 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:47:58.356 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:47:58.356 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:47:58.356 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:47:58.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:58.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.539 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:47:58.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:58.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:47:58.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:58.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:58.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:47:58.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:58.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:58.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:47:59.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:47:59.105 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 101576 101560 93.3351 77.6875 0.0105811 0.0107099 2024-09-13T14:47:59.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:47:59.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.142 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:59.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:47:59.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:47:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.287 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:59.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:47:59.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:47:59.437 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:47:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:59.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:47:59.524 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:47:59.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:59.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:47:59.753 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:47:59.753 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:47:59.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:47:59.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:47:59.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:47:59.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:47:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:47:59.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:47:59.847 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:47:59.848 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:47:59.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:47:59.879 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:47:59.879 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:47:59.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:47:59.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:47:59.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:47:59.905 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:47:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:47:59.935 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:47:59.936 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:47:59.944 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:59 smithi042 ceph-mon[113926]: pgmap v713: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:47:59.944 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:47:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3886627442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:59.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:59 smithi042 ceph-mon[108570]: pgmap v713: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:47:59.945 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:47:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3886627442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:47:59.945 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:47:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:47:59] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.51.0" 2024-09-13T14:47:59.975 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:47:59.975 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:47:59.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:59 smithi159 ceph-mon[102519]: pgmap v713: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:47:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:47:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3886627442' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:00.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:00.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:00.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 102700 102684 93.0004 70.25 0.0311112 0.0107475 2024-09-13T14:48:00.143 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:00.311 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:00.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:00.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:00.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:00.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:48:00.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:00.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:00.841 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:00.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:00.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:00.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:00.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.018 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/294132 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/294132 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:01.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 103942 103926 92.7807 77.625 0.0042346 0.0107732 2024-09-13T14:48:01.120 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:01.120 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/294132 objects degraded (1.408%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:01.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:01.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:01.255 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:01.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:01.391 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.432 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:01.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:01.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:01.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:01.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:01.616 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:01.617 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:01.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:01.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:01.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:01.834 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:01.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:01.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:01.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:01.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:01.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:02.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:01 smithi042 ceph-mon[113926]: pgmap v714: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:02.024 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:01 smithi042 ceph-mon[108570]: pgmap v714: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:02.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:02.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 105127 105111 92.517 74.0625 0.025644 0.0108041 2024-09-13T14:48:02.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:02.221 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:02.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:01 smithi159 ceph-mon[102519]: pgmap v714: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s; 4142/294132 objects degraded (1.408%); 4091/294132 objects misplaced (1.391%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:02.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:02.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:02.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:48:02.399 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:02.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:02.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:02.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:02.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:02.757 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:02.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:02.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:02.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:02.938 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:02.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:02.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:03.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:03.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 106302 106286 92.2519 73.4375 0.0170195 0.0108349 2024-09-13T14:48:03.108 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:03.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:03.251 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:03.289 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:03.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:03.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:03.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:03.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:03.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:03.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-09-13T14:48:03.637 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:03.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:03.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:03.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:03.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:03.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:03.912 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:03.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:03.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:04.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:04.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:03 smithi042 ceph-mon[108570]: pgmap v715: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 4142/310404 objects degraded (1.334%); 4091/310404 objects misplaced (1.318%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:03 smithi042 ceph-mon[113926]: pgmap v715: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 4142/310404 objects degraded (1.334%); 4091/310404 objects misplaced (1.318%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:04.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 107514 107498 92.0254 75.75 0.00856039 0.0108619 2024-09-13T14:48:04.135 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:03 smithi159 ceph-mon[102519]: pgmap v715: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 4142/310404 objects degraded (1.334%); 4091/310404 objects misplaced (1.318%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:48:04.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:04.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:04.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:04.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:04.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:04.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:04.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:04.700 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:04.713 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:04.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:04.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.831 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:04.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:04.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:04.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:04.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:04.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.046 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:05.046 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:05.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:05.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:05.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:05.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 108638 108622 91.7311 70.25 0.00703159 0.0108967 2024-09-13T14:48:05.200 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:05.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:05.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:05.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:05.463 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:05.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:05.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:05.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:05.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:05.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:05.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:05.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:05.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:05.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:05.908 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:48:05.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:05.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:06.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:06.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: pgmap v716: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 4142/312129 objects degraded (1.327%); 3427/312129 objects misplaced (1.098%); 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/312129 objects degraded (1.327%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3195344100' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: pgmap v716: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 4142/312129 objects degraded (1.327%); 3427/312129 objects misplaced (1.098%); 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/312129 objects degraded (1.327%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3195344100' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:06.101 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 109688 109672 91.3829 65.625 0.0045832 0.0109346 2024-09-13T14:48:06.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:06.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:06.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: pgmap v716: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.3 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 4142/312129 objects degraded (1.327%); 3427/312129 objects misplaced (1.098%); 4.0 MiB/s, 64 objects/s recovering 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/312129 objects degraded (1.327%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3195344100' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:48:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:06.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:06.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:06.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:06.345 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:48:06.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:06.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:06.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:06.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.638 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:06.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:06.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:06.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:06.783 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:06.878 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:06.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:06.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:06.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[108570]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:06.999 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:06 smithi042 ceph-mon[113926]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:48:07.020 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:07.054 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:07.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 110767 110751 91.0678 67.4375 0.01763 0.0109754 2024-09-13T14:48:07.147 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:06 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:48:07.147 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:06 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:48:07.147 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:06 smithi159 ceph-mon[102519]: osdmap e207: 8 total, 8 up, 7 in 2024-09-13T14:48:07.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:07.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:07.289 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:07.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:07.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:07.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:07.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:07.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:07.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:07.718 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:07.820 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:07.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:07.962 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:07.998 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:07.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:08.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 111966 111950 90.8582 74.9375 0.00602415 0.0110013 2024-09-13T14:48:08.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:07 smithi159 ceph-mon[102519]: pgmap v718: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/323526 objects degraded (1.280%); 3427/323526 objects misplaced (1.059%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:08.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:07 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:08.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:07 smithi042 ceph-mon[108570]: pgmap v718: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/323526 objects degraded (1.280%); 3427/323526 objects misplaced (1.059%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:08.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:07 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:08.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:07 smithi042 ceph-mon[113926]: pgmap v718: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.6 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 4142/323526 objects degraded (1.280%); 3427/323526 objects misplaced (1.059%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:08.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:07 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:08.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:08.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:08.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:08.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:08.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:08.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.704 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:08.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:08.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:08.851 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:08.938 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:08.976 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:08.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:09.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 15 113091 113076 90.5955 70.375 0.012938 0.0110332 2024-09-13T14:48:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:09.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:09.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:09.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:09.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:09.518 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:09.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:48:09.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:09.657 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:09.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:09.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:09.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:09] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.51.0" 2024-09-13T14:48:09.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:09.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:09.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:09.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:09.978 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:48:10.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:10.095 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:10.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:10.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 114219 114203 90.3403 70.4375 0.0137832 0.0110641 2024-09-13T14:48:10.167 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:09 smithi159 ceph-mon[102519]: pgmap v719: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:10.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:09 smithi042 ceph-mon[113926]: pgmap v719: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:10.183 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:09 smithi042 ceph-mon[108570]: pgmap v719: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:10.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:10.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:10.253 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:48:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:10.282 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:48:10.282 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:10.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:48:10.307 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:10.338 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:48:10.338 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:10.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:10.365 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:48:10.366 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:10.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:10.411 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:48:10.411 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:10.442 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:48:10.442 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:10.477 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:48:10.477 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:10.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:10.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:10.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:10.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:10.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:48:10.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:10.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:10.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:10.813 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:10.915 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:10.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:10.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:10.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:10.993 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:10 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2468140541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:10.994 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 4142/329310 objects degraded (1.258%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:10.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:10 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2468140541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:10.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 4142/329310 objects degraded (1.258%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:11.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:11.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:11.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:11.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:48:11.102275+0000 min lat: 0.0013466 max lat: 2.54826 avg lat: 0.0110903 2024-09-13T14:48:11.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:48:11.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 115397 115381 90.1313 73.625 0.00588842 0.0110903 2024-09-13T14:48:11.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:11.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:11.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:10 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2468140541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:11.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 4142/329310 objects degraded (1.258%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:11.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:11.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:11.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:11.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:11.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:11.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:11.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:11.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:11.897 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:11.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:11 smithi159 ceph-mon[102519]: pgmap v720: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:11 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:11.999 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:12.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:11 smithi042 ceph-mon[113926]: pgmap v720: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:11 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:11 smithi042 ceph-mon[108570]: pgmap v720: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 6.7 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 4142/329310 objects degraded (1.258%); 2703/329310 objects misplaced (0.821%); 4.3 MiB/s, 69 objects/s recovering 2024-09-13T14:48:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:11 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:12.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 116401 116385 89.7932 62.75 0.0180132 0.0111301 2024-09-13T14:48:12.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:12.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:12.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:12.260 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:12.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:12.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:12.415 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:12.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:12.539 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:12.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:12.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:12.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:12.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:12.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:12.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:12.887 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:12.909 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:12.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:13.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:13.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 116755 116739 88.9679 22.125 0.0158122 0.0112354 2024-09-13T14:48:13.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:13.197 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:12 smithi159 ceph-mon[102519]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:48:13.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:13.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:12 smithi042 ceph-mon[108570]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:48:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:12 smithi042 ceph-mon[113926]: osdmap e208: 8 total, 8 up, 7 in 2024-09-13T14:48:13.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.453 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:13.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:13.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:13.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:13.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:13.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:13.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:13.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:48:13.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:13.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:13.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:13.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:14.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:14.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[113926]: pgmap v722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 4142/344769 objects degraded (1.201%); 2703/344769 objects misplaced (0.784%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:48:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[113926]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:48:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[108570]: pgmap v722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 4142/344769 objects degraded (1.201%); 2703/344769 objects misplaced (0.784%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:48:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[108570]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:48:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:13 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:14.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 117780 117764 88.6678 64.0625 0.00515276 0.0112731 2024-09-13T14:48:14.119 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:14.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:13 smithi159 ceph-mon[102519]: pgmap v722: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s; 4142/344769 objects degraded (1.201%); 2703/344769 objects misplaced (0.784%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:48:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:13 smithi159 ceph-mon[102519]: osdmap e209: 8 total, 8 up, 7 in 2024-09-13T14:48:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:13 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:14.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:14.289 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:14.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:14.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:14.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:14.527 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:14.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:14.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:14.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:14.778 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:14.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:14.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:14.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-09-13T14:48:14.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:14.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:14.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:15.023 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:15.061 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:15.082 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:48:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:48:15.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 118707 118691 88.3019 57.9375 0.0262483 0.0113189 2024-09-13T14:48:15.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:15.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:15.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"} : dispatch 2024-09-13T14:48:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.285 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:15.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:15.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:15.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:15.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:15.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:15.531 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:15.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:15.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:15.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:15.893 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:15.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:15.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:16.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:16.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 119245 119229 87.6586 33.625 0.00285398 0.011361 2024-09-13T14:48:16.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:16.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: pgmap v724: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 6557/345726 objects misplaced (1.897%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4142/344769 objects degraded (1.201%), 1 pg degraded) 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:48:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:15 smithi159 ceph-mon[102519]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:48:16.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:16.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:16.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: pgmap v724: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 6557/345726 objects misplaced (1.897%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4142/344769 objects degraded (1.201%), 1 pg degraded) 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:48:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[113926]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: pgmap v724: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s; 6557/345726 objects misplaced (1.897%); 3.6 MiB/s, 57 objects/s recovering 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4142/344769 objects degraded (1.201%), 1 pg degraded) 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: osdmap e210: 8 total, 8 up, 7 in 2024-09-13T14:48:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:15 smithi042 ceph-mon[108570]: osdmap e211: 8 total, 8 up, 7 in 2024-09-13T14:48:16.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:16.447 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:16.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:16.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:16.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:16.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:16.674 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:16.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:16.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:16.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:16.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:16.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:16.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:16.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:16.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.102 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 119473 119457 86.8051 14.25 0.0106783 0.0115144 2024-09-13T14:48:17.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:17.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84558 9.3 9:c85b4db8:::benchmark_data_smithi042_157727_object84557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:48:17.178 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.38803495299999, 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.37831424299999999, 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84558 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:48:17.179 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641438+0000", 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.180 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641520+0000", 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2422000000000005e-05 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641531+0000", 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0536e-05 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.181 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641537+0000", 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642039+0000", 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050194900000000001 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642168+0000", 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012866399999999999 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.182 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642290+0000", 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012242000000000001 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.647958+0000", 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056680840000000003 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.183 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719589+0000", 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.071630894000000001 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019655+0000", 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30006546099999998 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.184 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019711+0000", 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6632000000000003e-05 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019753+0000", 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1925999999999998e-05 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84559 9.3 9:c8ad0e78:::benchmark_data_smithi042_157727_object84558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.185 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.38788756299999, 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.38015051100000002, 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84559 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.186 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:17.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641983+0000", 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039788200000000001 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642357+0000", 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00037351099999999998 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642364+0000", 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2790000000000001e-06 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645661+0000", 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032972330000000001 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645784+0000", 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012222900000000001 2024-09-13T14:48:17.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645882+0000", 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8276999999999994e-05 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.648108+0000", 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022264009999999998 2024-09-13T14:48:17.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.722650+0000", 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074541741999999994 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021646+0000", 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29899620700000001 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021712+0000", 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5488999999999996e-05 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021737+0000", 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4961e-05 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.193 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84563 9.d 9:ba52b3ee:::benchmark_data_smithi042_157727_object84562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.30970912200002, 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.31030390200000002, 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84563 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719763+0000", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719824+0000", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0915999999999997e-05 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719836+0000", 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1992e-05 2024-09-13T14:48:17.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719841+0000", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.271e-06 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719885+0000", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3130000000000002e-05 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719994+0000", 2024-09-13T14:48:17.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109428 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.720091+0000", 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7001000000000002e-05 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723173+0000", 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030821899999999998 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723401+0000", 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022781400000000001 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.029965+0000", 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30656361599999998 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030017+0000", 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2584999999999999e-05 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030069+0000", 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1329999999999998e-05 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84572 9.3 9:cea63616:::benchmark_data_smithi042_157727_object84571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.30187259299998, 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30071340699999999, 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.200 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84572 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727600+0000", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727712+0000", 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112267 2024-09-13T14:48:17.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727728+0000", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6545000000000002e-05 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727736+0000", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7950000000000008e-06 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727777+0000", 2024-09-13T14:48:17.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0595e-05 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727921+0000", 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000143879 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728044+0000", 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000123196 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730658+0000", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026146229999999999 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731148+0000", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048908600000000001 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028092+0000", 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29694452599999999 2024-09-13T14:48:17.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028269+0000", 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000177126 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028315+0000", 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5414999999999998e-05 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84573 9.e 9:7570e4c0:::benchmark_data_smithi042_157727_object84572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.30168399799999, 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29393596500000002, 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84573 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727789+0000", 2024-09-13T14:48:17.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727845+0000", 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5803000000000001e-05 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727853+0000", 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3659999999999992e-06 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727857+0000", 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9099999999999998e-06 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727888+0000", 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0574999999999998e-05 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727979+0000", 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1242999999999994e-05 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728059+0000", 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0238e-05 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730776+0000", 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002716684 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730964+0000", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018814 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021615+0000", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29065137200000002 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021666+0000", 2024-09-13T14:48:17.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0467000000000003e-05 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021726+0000", 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9982000000000003e-05 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84574 9.d 9:bb2c6928:::benchmark_data_smithi042_157727_object84573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 315.29837196400001, 2024-09-13T14:48:17.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29919750000000001, 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84574 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731101+0000", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731161+0000", 2024-09-13T14:48:17.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0677999999999997e-05 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731171+0000", 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7480000000000005e-06 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731177+0000", 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731204+0000", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7103000000000001e-05 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731302+0000", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7919999999999998e-05 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731399+0000", 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6759e-05 2024-09-13T14:48:17.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733667+0000", 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022682940000000001 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733966+0000", 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000298528 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030141+0000", 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29617486399999998 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030264+0000", 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012284000000000001 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030299+0000", 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5714999999999999e-05 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.219 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111462 9.d 9:be6e2a9e:::benchmark_data_smithi042_157727_object111461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 290.454173744, 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.090455042999999999, 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111462 2024-09-13T14:48:17.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575300+0000", 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575299+0000", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575330+0000", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1720000000000001e-05 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575344+0000", 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3326000000000001e-05 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575350+0000", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7780000000000002e-06 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575386+0000", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5271000000000002e-05 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575538+0000", 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015224600000000001 2024-09-13T14:48:17.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575677+0000", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000138925 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.579009+0000", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00333228 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.579562+0000", 2024-09-13T14:48:17.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000552773 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665646+0000", 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.086084385999999999 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665695+0000", 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8665999999999998e-05 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.665755+0000", 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0127000000000002e-05 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111463 9.3 9:c145cb7d:::benchmark_data_smithi042_157727_object111462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 290.45406423600002, 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.086132666999999996, 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111463 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575410+0000", 2024-09-13T14:48:17.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575409+0000", 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575476+0000", 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7002999999999995e-05 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575484+0000", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3489999999999992e-06 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575490+0000", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7660000000000004e-06 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575529+0000", 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9245000000000001e-05 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575610+0000", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0665000000000001e-05 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.575695+0000", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4841999999999998e-05 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.578801+0000", 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003106765 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.582252+0000", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034510109999999999 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661446+0000", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.079193685999999999 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661489+0000", 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2667000000000003e-05 2024-09-13T14:48:17.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.661542+0000", 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3461999999999997e-05 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111477 9.e 9:7ca6fc8e:::benchmark_data_smithi042_157727_object111476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 290.44125154, 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.085265375000000004, 2024-09-13T14:48:17.232 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111477 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588222+0000", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588221+0000", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588281+0000", 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0575000000000002e-05 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588294+0000", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2473000000000001e-05 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588298+0000", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7929999999999997e-06 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588330+0000", 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1347999999999999e-05 2024-09-13T14:48:17.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588456+0000", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012629699999999999 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588548+0000", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1544000000000003e-05 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.591582+0000", 2024-09-13T14:48:17.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030342239999999999 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.592407+0000", 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00082471300000000003 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673441+0000", 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.081034410000000001 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673469+0000", 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7577e-05 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.673488+0000", 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9074999999999999e-05 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111478 9.3 9:c13653fd:::benchmark_data_smithi042_157727_object111477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:48:17.238 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 290.44112897999997, 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.096331477999999998, 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111478 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588345+0000", 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588344+0000", 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588419+0000", 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4846999999999997e-05 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588428+0000", 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7290000000000006e-06 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588434+0000", 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2820000000000002e-06 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588480+0000", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5836e-05 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588639+0000", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015876700000000001 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.588790+0000", 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015128799999999999 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.591754+0000", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0029641699999999999 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.672268+0000", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.080513578000000002 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684565+0000", 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012296860999999999 2024-09-13T14:48:17.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684637+0000", 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2665999999999998e-05 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:26.684676+0000", 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9135999999999999e-05 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111884 9.3 9:c842fdc2:::benchmark_data_smithi042_157727_object111883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 289.679934011, 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.108810141, 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111884 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349540+0000", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349538+0000", 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:48:17.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349608+0000", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0049000000000006e-05 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349624+0000", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5537999999999999e-05 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349631+0000", 2024-09-13T14:48:17.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8120000000000008e-06 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349675+0000", 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3405000000000002e-05 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.349870+0000", 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000195513 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.350096+0000", 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000225796 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.354559+0000", 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044625400000000001 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.354567+0000", 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4570000000000005e-06 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458271+0000", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.103703352 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458315+0000", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4193000000000002e-05 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.458350+0000", 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5204999999999999e-05 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.250 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:111892 9.3 9:cc204ca5:::benchmark_data_smithi042_157727_object111891:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 289.67215220600002, 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.106581274, 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:48:17.251 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 111892 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357322+0000", 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357320+0000", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357396+0000", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5364000000000006e-05 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357408+0000", 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2428e-05 2024-09-13T14:48:17.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357413+0000", 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3120000000000001e-06 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357455+0000", 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.193e-05 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357560+0000", 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000105042 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.357679+0000", 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011904399999999999 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.360725+0000", 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030456329999999998 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.361549+0000", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00082388600000000002 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463812+0000", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.102262805 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463861+0000", 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9280000000000003e-05 2024-09-13T14:48:17.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:27.463903+0000", 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1993000000000003e-05 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.34797.0:235007 9.7 e173/139 9:ec4e1241:::benchmark_data_smithi042_157727_object105058:head v 173'14666, mlcod=173'14664)", 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:45:36.447542+0000", 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 160.58193186400001, 2024-09-13T14:48:17.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.093380529000000004, 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.447542+0000", 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.447542+0000", 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.447541+0000", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.447549+0000", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0760000000000003e-06 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.518343+0000", 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.070794037000000004 2024-09-13T14:48:17.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.518356+0000", 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3021e-05 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.518576+0000", 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022013499999999999 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.518601+0000", 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4762000000000001e-05 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.540889+0000", 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022287863000000002 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:45:36.540922+0000", 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3735000000000001e-05 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.261 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:39878 10.c e189/184 10:327dafc5:::benchmark_data_smithi042_191918_object39877:head v 189'2481, mlcod=189'2479)", 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:06.122017+0000", 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 70.907456488999998, 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.085701204000000003, 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122017+0000", 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122017+0000", 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122016+0000", 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122040+0000", 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3570999999999998e-05 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122052+0000", 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1581e-05 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122058+0000", 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0480000000000004e-06 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122102+0000", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4019e-05 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.122124+0000", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2710000000000001e-05 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.207684+0000", 2024-09-13T14:48:17.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.085559489000000002 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:06.207719+0000", 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4659000000000001e-05 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96791 10.c e205/184 10:3cfbffb8:::benchmark_data_smithi042_191918_object96790:head v 205'6009, mlcod=205'6007)", 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:48:17.266 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.060177546999999, 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16168563899999999, 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969295+0000", 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969317+0000", 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1963e-05 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969329+0000", 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1556000000000001e-05 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969335+0000", 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2720000000000003e-06 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969400+0000", 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4843000000000002e-05 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969419+0000", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9174999999999999e-05 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130936+0000", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.161517046 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130982+0000", 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5559000000000002e-05 2024-09-13T14:48:17.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96796 10.c e205/184 10:3e3d83db:::benchmark_data_smithi042_191918_object96795:head v 205'6010, mlcod=205'6007)", 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.056749653000001, 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.15827959899999999, 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972723+0000", 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972747+0000", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3302000000000002e-05 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972757+0000", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0682e-05 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972764+0000", 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5060000000000001e-06 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972822+0000", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8304999999999998e-05 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972841+0000", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9266000000000002e-05 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130986+0000", 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.158144854 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131004+0000", 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7623999999999999e-05 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96798 10.15 e205/192 10:ab2d6cee:::benchmark_data_smithi042_191918_object96797:head v 205'4774, mlcod=205'4772)", 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.055869711, 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.160394434, 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.275 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973603+0000", 2024-09-13T14:48:17.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973649+0000", 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5407000000000001e-05 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973660+0000", 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1304999999999999e-05 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973667+0000", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.759e-06 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973725+0000", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7856999999999998e-05 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973744+0000", 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9891999999999999e-05 2024-09-13T14:48:17.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.133917+0000", 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16017301 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.133999+0000", 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1118000000000005e-05 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96803 10.18 e205/192 10:1bbb31d9:::benchmark_data_smithi042_191918_object96802:head v 205'4706, mlcod=205'4704)", 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.054729700999999, 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16000645999999999, 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974743+0000", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974784+0000", 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0402999999999999e-05 2024-09-13T14:48:17.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130893+0000", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.156109523 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130910+0000", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7244999999999998e-05 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131222+0000", 2024-09-13T14:48:17.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00031179800000000001 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131240+0000", 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7680999999999999e-05 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134683+0000", 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003443345 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134751+0000", 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7281999999999997e-05 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96804 10.19 e205/192 10:9bbfbfa6:::benchmark_data_smithi042_191918_object96803:head v 205'4741, mlcod=205'4739)", 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.049819883000001, 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.15497003200000001, 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.284 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979653+0000", 2024-09-13T14:48:17.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979677+0000", 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4644e-05 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130894+0000", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.151217142 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130910+0000", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5679999999999999e-05 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131118+0000", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020829099999999999 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131138+0000", 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9162999999999999e-05 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134541+0000", 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00340348 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134624+0000", 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2886999999999994e-05 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.288 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96811 10.c e205/184 10:3bccbb40:::benchmark_data_smithi042_191918_object96810:head v 205'6011, mlcod=205'6007)", 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 22.046068073000001, 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.153947004, 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:48:17.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983405+0000", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983453+0000", 2024-09-13T14:48:17.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8809e-05 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131021+0000", 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14756804700000001 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131029+0000", 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1760000000000004e-06 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:48:17.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131066+0000", 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7713999999999998e-05 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131085+0000", 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9096000000000001e-05 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.137207+0000", 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061217750000000003 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:48:17.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.137353+0000", 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014560300000000001 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:48:17.293 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:48:17.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:17.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.476 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:17.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:17 smithi042 ceph-mon[113926]: pgmap v727: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 7.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 5925/354075 objects misplaced (1.673%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:17 smithi042 ceph-mon[113926]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:48:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:17 smithi042 ceph-mon[108570]: pgmap v727: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 7.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 5925/354075 objects misplaced (1.673%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:17 smithi042 ceph-mon[108570]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:48:17.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:17.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:17 smithi159 ceph-mon[102519]: pgmap v727: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 7.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 5925/354075 objects misplaced (1.673%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:17 smithi159 ceph-mon[102519]: osdmap e212: 8 total, 8 up, 7 in 2024-09-13T14:48:17.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:17.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:17.865 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:17.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:17.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:17.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:18.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 120575 120559 86.5989 68.875 0.0118514 0.0115428 2024-09-13T14:48:18.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:18.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:18.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:48:18.284 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:18.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:18.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:18.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:18.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:18.567 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:18.617 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:18.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:18.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:18.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:18.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:18.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:18.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:18.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:19.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:19.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 121571 121555 86.3221 62.25 0.0194754 0.0115796 2024-09-13T14:48:19.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.159 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.260 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:19.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:19.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:19.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:19 smithi042 ceph-mon[113926]: pgmap v729: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 5286/357582 objects misplaced (1.478%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:48:19.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:19 smithi042 ceph-mon[108570]: pgmap v729: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 5286/357582 objects misplaced (1.478%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:48:19.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:19.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:19 smithi159 ceph-mon[102519]: pgmap v729: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 50 MiB/s wr, 794 op/s; 5286/357582 objects misplaced (1.478%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:48:19.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:19.850 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:19.953 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:19.998 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:19.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:19.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:19] "GET /metrics HTTP/1.1" 200 36469 "" "Prometheus/2.51.0" 2024-09-13T14:48:20.100 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:20.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 122714 122698 86.1548 71.4375 0.00865529 0.011602 2024-09-13T14:48:20.125 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:20.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:20.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.263 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:20.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:20.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:20.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:20.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:20.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:20.539 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:20.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:20.641 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.682 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:20.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:20 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:20.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:20.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:20.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:20.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:20 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:20.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:20 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:20.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:20.949 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:20.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:21.051 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:21.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:21.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:21.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 90 11 123590 123579 85.8093 55.0625 0.0936533 0.0116424 2024-09-13T14:48:21.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:21.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:21.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:21.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:21.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:21.424 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:21.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:21.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:21.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:21.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:21.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:21.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:21.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:21 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:21 smithi159 ceph-mon[102519]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 5286/357582 objects misplaced (1.478%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:21.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:21.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[113926]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 5286/357582 objects misplaced (1.478%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[108570]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 remapped+peering, 107 active+clean; 7.3 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 41 MiB/s wr, 658 op/s; 5286/357582 objects misplaced (1.478%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:21.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:21.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:21.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:22.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:22.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:22.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.019 2024-09-13T14:48:22.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 123590 2024-09-13T14:48:22.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:48:22.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:48:22.103 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 85.8083 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 46.3475 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 243.062 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1372 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 741.569 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3889 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 0 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0116493 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0450896 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 2.54826 2024-09-13T14:48:22.104 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.0013466 2024-09-13T14:48:22.109 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:48:22.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:22.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:22.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:22.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.370 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:22.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:22.471 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:22.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:22.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:22.608 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:22.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:22.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:22.735 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:22.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:22 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:22.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:48:22.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:22.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:22.847 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:22 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:22.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:48:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:22 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-13T14:48:22.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:22.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:22.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:23.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:23.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:23.152 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:23.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:23.282 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:23.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:23.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:23.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:23.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:23.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:23.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:23.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:23.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:23 smithi159 ceph-mon[102519]: pgmap v731: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 106 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 12413/369759 objects misplaced (3.357%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:48:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:23 smithi159 ceph-mon[102519]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:48:23.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:23.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[108570]: pgmap v731: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 106 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 12413/369759 objects misplaced (3.357%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:48:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[108570]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:48:23.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[113926]: pgmap v731: 109 pgs: 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 106 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 12413/369759 objects misplaced (3.357%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:23.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-13T14:48:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:23 smithi042 ceph-mon[113926]: osdmap e213: 8 total, 8 up, 7 in 2024-09-13T14:48:23.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:23.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:23.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:24.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:24.036 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:24.138 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:24.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:24.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:24.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:24.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:24.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:24.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:24.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:24.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:24.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:24 smithi042 ceph-mon[108570]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:48:24.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:24 smithi042 ceph-mon[113926]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:48:24.637 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:24.669 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:24.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:24 smithi159 ceph-mon[102519]: osdmap e214: 8 total, 8 up, 7 in 2024-09-13T14:48:24.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:24.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:24.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:24.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:24.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:24.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:25.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:25.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:25.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.221 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:25.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:25.357 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:25.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.597 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:25.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:25.736 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:25 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:25 smithi159 ceph-mon[102519]: pgmap v734: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 11621/370086 objects misplaced (3.140%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:25 smithi159 ceph-mon[102519]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:48:25.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[113926]: pgmap v734: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 11621/370086 objects misplaced (3.140%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[113926]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:48:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[108570]: pgmap v734: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.5 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 43 MiB/s wr, 691 op/s; 11621/370086 objects misplaced (3.140%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:25 smithi042 ceph-mon[108570]: osdmap e215: 8 total, 8 up, 7 in 2024-09-13T14:48:25.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:25.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:25.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:25.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:26.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:26.048 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:26.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:26.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.329 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:26.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:26.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:26.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:26.573 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:26.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:26.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:26.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:26.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:26 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:26.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:26.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:26 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:26.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:26 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:26.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:26.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:26.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:26.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:26.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.022 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:27.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.167 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:27.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:27.268 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:27.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:27.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:27.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:27.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:27.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:27.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.577 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:27.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:27.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:27 smithi042 ceph-mon[108570]: pgmap v736: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.4 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 45 MiB/s wr, 1.07k op/s; 11134/364098 objects misplaced (3.058%); 6.6 MiB/s, 104 objects/s recovering 2024-09-13T14:48:27.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:27 smithi042 ceph-mon[113926]: pgmap v736: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.4 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 45 MiB/s wr, 1.07k op/s; 11134/364098 objects misplaced (3.058%); 6.6 MiB/s, 104 objects/s recovering 2024-09-13T14:48:27.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:27.708 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:27.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:27.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:27 smithi159 ceph-mon[102519]: pgmap v736: 109 pgs: 1 unknown, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 105 active+clean; 7.4 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 45 MiB/s wr, 1.07k op/s; 11134/364098 objects misplaced (3.058%); 6.6 MiB/s, 104 objects/s recovering 2024-09-13T14:48:27.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:27.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:27.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:27.976 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:27.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:28.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:28.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:28.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:28.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:28.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:28.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:28.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:28.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:28.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:28.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:28.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:28 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 3857/360639 objects degraded (1.069%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:28.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:28.800 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:28.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:28 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 3857/360639 objects degraded (1.069%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:28.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:28 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 3857/360639 objects degraded (1.069%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:48:28.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:28.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:28.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:28.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:29.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:29.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:29.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:29.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:29.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:29.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:29.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:29.728 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:29 smithi042 ceph-mon[113926]: pgmap v737: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 2.4 MiB/s wr, 584 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:48:29.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:29 smithi042 ceph-mon[108570]: pgmap v737: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 2.4 MiB/s wr, 584 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:48:29.729 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:29.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:29 smithi159 ceph-mon[102519]: pgmap v737: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 170 B/s rd, 2.4 MiB/s wr, 584 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 8.1 MiB/s, 129 objects/s recovering 2024-09-13T14:48:29.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:29.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:29.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:29.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:29.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:29.998 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:30.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:30.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:29] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.51.0" 2024-09-13T14:48:30.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.300 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:30.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:30.538 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:30.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:30.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:30.702 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:30 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:48:30.702 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:30 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:48:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:30.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:30 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:48:30.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:30.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:30.966 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:30.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:30.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:30.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:31.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:31.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:31.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:31.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:31.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:31.543 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:31.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:31.645 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:31.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.680 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:31.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:31 smithi159 ceph-mon[102519]: 10.10 deep-scrub starts 2024-09-13T14:48:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:31 smithi159 ceph-mon[102519]: pgmap v738: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 128 B/s rd, 1.8 MiB/s wr, 441 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:48:31.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:31.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:31.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:31.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:31 smithi042 ceph-mon[113926]: 10.10 deep-scrub starts 2024-09-13T14:48:31.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:31 smithi042 ceph-mon[113926]: pgmap v738: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 128 B/s rd, 1.8 MiB/s wr, 441 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:48:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:31 smithi042 ceph-mon[108570]: 10.10 deep-scrub starts 2024-09-13T14:48:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:31 smithi042 ceph-mon[108570]: pgmap v738: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 128 B/s rd, 1.8 MiB/s wr, 441 op/s; 3857/360639 objects degraded (1.069%); 10333/360639 objects misplaced (2.865%); 6.1 MiB/s, 97 objects/s recovering 2024-09-13T14:48:31.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:31.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:31.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:32.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:32.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:32.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:32.272 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:32.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:32.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:32.512 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:32.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:32.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:32.651 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:32.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:32.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:32.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:32.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:32.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:32.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:32.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:32.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:33.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:33.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:33.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:33.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:33.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:33.361 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:33.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:33.463 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.514 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:33.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:33.533 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:33.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:33.651 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:33.659 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:33 smithi042 ceph-mon[113926]: pgmap v739: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 114 B/s rd, 265 KiB/s wr, 919 op/s; 3857/345657 objects degraded (1.116%); 10037/345657 objects misplaced (2.904%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:33.660 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:33 smithi042 ceph-mon[108570]: pgmap v739: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 114 B/s rd, 265 KiB/s wr, 919 op/s; 3857/345657 objects degraded (1.116%); 10037/345657 objects misplaced (2.904%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:33.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:33 smithi159 ceph-mon[102519]: pgmap v739: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 114 B/s rd, 265 KiB/s wr, 919 op/s; 3857/345657 objects degraded (1.116%); 10037/345657 objects misplaced (2.904%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:33.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:33.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:33.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:33.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:33.923 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:33.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:33.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:34.025 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:34.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:34.202 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:48:34.202 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:34.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:34.243 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:48:34.243 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:34.279 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:48:34.280 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:34.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:34.312 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:48:34.312 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:34.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:48:34.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:34.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:34.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:48:34.386 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:34.428 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:48:34.428 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:34.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.474 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:48:34.474 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:48:34.475 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:34.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:34.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1222971858' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3598680200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.566 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1222971858' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.567 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.567 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3598680200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.639 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:34.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:34.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:34.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1222971858' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:34 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:48:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3598680200' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:34.780 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:34.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:34.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:34.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.023 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:35.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:35.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:35.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:35.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:35.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:35.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:35.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:35.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:35.620 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:35.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:35.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: pgmap v740: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 237 KiB/s wr, 877 op/s; 3857/343992 objects degraded (1.121%); 9433/343992 objects misplaced (2.742%); 5.4 MiB/s, 85 objects/s recovering 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3857/343992 objects degraded (1.121%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:48:35.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:35.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:35.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:35.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: pgmap v740: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 237 KiB/s wr, 877 op/s; 3857/343992 objects degraded (1.121%); 9433/343992 objects misplaced (2.742%); 5.4 MiB/s, 85 objects/s recovering 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3857/343992 objects degraded (1.121%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:48:35.834 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: pgmap v740: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 237 KiB/s wr, 877 op/s; 3857/343992 objects degraded (1.121%); 9433/343992 objects misplaced (2.742%); 5.4 MiB/s, 85 objects/s recovering 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3857/343992 objects degraded (1.121%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: osdmap e216: 8 total, 8 up, 7 in 2024-09-13T14:48:35.835 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:48:35.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:35.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:35.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:36.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:36.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:36.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:36.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:36.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:36.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:36.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:36.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:36.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:36.509 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:36.611 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:36.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:36.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:36.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:36.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:36.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:36.941 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:36.941 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:36.941 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:36.941 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:36.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:36.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:37.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:37.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:37.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:37.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:37.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:37.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:37.365 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:37.467 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:37.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.600 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:37.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:37.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:37.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:37.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:37.781 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:37 smithi042 ceph-mon[108570]: pgmap v742: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/332229 objects degraded (1.161%); 9117/332229 objects misplaced (2.744%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:48:37.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:37.898 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:37.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:37.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:37.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:37 smithi159 ceph-mon[102519]: pgmap v742: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/332229 objects degraded (1.161%); 9117/332229 objects misplaced (2.744%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:48:38.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:38.032 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:37 smithi042 ceph-mon[113926]: pgmap v742: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/332229 objects degraded (1.161%); 9117/332229 objects misplaced (2.744%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:48:38.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.187 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:38.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:38.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:38.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:38.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:38.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:38.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:38.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:48:38.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:38.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:38.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:38.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:38.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:38.979 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:38.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:39.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:39.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:39.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:39.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:39.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:39.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:39.484 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:39.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:39.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:39.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:39.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:39.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:39.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:39.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:39.887 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:39.924 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:39.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:40.050 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:40.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:40.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:39 smithi042 ceph-mon[113926]: pgmap v743: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:39 smithi042 ceph-mon[108570]: pgmap v743: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:39] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-13T14:48:40.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:40.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:39 smithi159 ceph-mon[102519]: pgmap v743: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:40.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:40.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:40.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:40.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:40.431 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:40.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:40.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:40.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:40.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:40.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:40.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:40.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:40.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:40.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:40.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:40.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:40.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:41.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3857/327015 objects degraded (1.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3857/327015 objects degraded (1.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:41.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:41.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.210 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3857/327015 objects degraded (1.179%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:41.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:41.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:41.493 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:41.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:41.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:41.632 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:41.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:41.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.794 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:41.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:41.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:41.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:41.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:41.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:42.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.053 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:42.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:41 smithi042 ceph-mon[113926]: pgmap v744: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:41 smithi042 ceph-mon[108570]: pgmap v744: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:42.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:42.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:42.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:41 smithi159 ceph-mon[102519]: pgmap v744: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.7 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 3857/327015 objects degraded (1.179%); 8436/327015 objects misplaced (2.580%); 4.6 MiB/s, 73 objects/s recovering 2024-09-13T14:48:42.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:42.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:42.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:42.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:42.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:42.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:42.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:42.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:42.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:42.851 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:42.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:42.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:42.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:43.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:43.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:43.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:43.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:43.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:43.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:43.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:43.437 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:43.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:43.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:43.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:43.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:43.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:43.834 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:43.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:43.936 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:43.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:43.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:43.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:44.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:43 smithi042 ceph-mon[113926]: pgmap v745: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 3857/313359 objects degraded (1.231%); 8096/313359 objects misplaced (2.584%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:43 smithi042 ceph-mon[108570]: pgmap v745: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 3857/313359 objects degraded (1.231%); 8096/313359 objects misplaced (2.584%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:44.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:44.123 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:43 smithi159 ceph-mon[102519]: pgmap v745: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 3857/313359 objects degraded (1.231%); 8096/313359 objects misplaced (2.584%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:48:44.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:44.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:44.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:44.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:44.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:44.471 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.505 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:44.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:44.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:44.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:44.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:44.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:44.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:44.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:44.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:45.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:45.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:45.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:45.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:45.313 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:45.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:45.453 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:45.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:45.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:45.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:45.674 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:45 smithi159 ceph-mon[102519]: pgmap v746: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/312102 objects degraded (1.236%); 7407/312102 objects misplaced (2.373%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:48:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3857/312102 objects degraded (1.236%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:45.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:45.810 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:45.864 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[113926]: pgmap v746: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/312102 objects degraded (1.236%); 7407/312102 objects misplaced (2.373%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:48:45.864 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.864 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.865 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3857/312102 objects degraded (1.236%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:45.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[108570]: pgmap v746: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 3857/312102 objects degraded (1.236%); 7407/312102 objects misplaced (2.373%); 4.9 MiB/s, 78 objects/s recovering 2024-09-13T14:48:45.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:48:45.865 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3857/312102 objects degraded (1.236%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:45.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:45.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:45.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:46.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:46.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:46.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:46.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:46.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:46.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:46.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:46.746 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:46.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:46.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:46.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:46.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:47.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:47.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.047 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:47.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:47.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.189 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:47.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:47.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:47.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.374 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:48:47.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.481 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:47.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.560 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:48:47.560 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:48:47.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:47.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:47.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.757 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.808 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:47 smithi042 ceph-mon[113926]: pgmap v747: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 3857/301890 objects degraded (1.278%); 7029/301890 objects misplaced (2.328%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:48:47.808 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:47 smithi042 ceph-mon[108570]: pgmap v747: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 3857/301890 objects degraded (1.278%); 7029/301890 objects misplaced (2.328%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:48:47.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:47.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:47.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:47.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:47.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:47.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:47 smithi159 ceph-mon[102519]: pgmap v747: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.1 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 3857/301890 objects degraded (1.278%); 7029/301890 objects misplaced (2.328%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:48:48.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:48.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:48.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:48.173 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:48.274 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:48.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:48.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:48.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:48.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:48.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:48.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:48.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:48:48.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:48.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:48.753 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:48.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:48.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:48.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:48.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.002 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:49.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.022 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:49.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.123 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:49.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:49.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:49.262 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:49.303 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:49.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:49.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:49.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:49.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:49.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:49.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:49.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:49.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:49.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:49.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:49.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:49.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:49.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:49 smithi159 ceph-mon[102519]: pgmap v748: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:48:50.008 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:49 smithi042 ceph-mon[108570]: pgmap v748: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:48:50.008 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:49 smithi042 ceph-mon[113926]: pgmap v748: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 966 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 5.5 MiB/s, 87 objects/s recovering 2024-09-13T14:48:50.009 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:49] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-13T14:48:50.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:50.140 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:50.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:50.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:50.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:50.411 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:50.513 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.532 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:50.591 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:50.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:50.697 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3857/297417 objects degraded (1.297%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:50.698 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3857/297417 objects degraded (1.297%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:50.730 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:50.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:50.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:50.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:50.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:50.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:50.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3857/297417 objects degraded (1.297%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:51.010 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:51.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:51.145 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:51.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:51.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:51.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:51.386 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:48:51.387 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:48:51.421 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:51.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:51.423 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:48:51.423 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:48:51.456 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:48:51.456 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:48:51.483 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:48:51.483 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:48:51.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:51.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:48:51.524 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:48:51.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:51.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:51.563 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:48:51.564 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:48:51.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:48:51.602 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:48:51.640 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:48:51.640 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:48:51.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:51.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:51.793 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:51.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:51.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[113926]: pgmap v749: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3958858958' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[108570]: pgmap v749: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:51.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3958858958' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:51.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:51.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:51.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:51.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:51 smithi159 ceph-mon[102519]: pgmap v749: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 6.0 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 821 op/s; 3857/297417 objects degraded (1.297%); 6329/297417 objects misplaced (2.128%); 4.3 MiB/s, 68 objects/s recovering 2024-09-13T14:48:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:48:51.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3958858958' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:48:52.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:52.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:52.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:52.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:52.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:52.347 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.380 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:52.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:52.481 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:52.514 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:52.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:52.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:52.648 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:52.750 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:52.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:52.887 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:52.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:52.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:52.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:53.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:53.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:53.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:53.322 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:53.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:53.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:53.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:53.548 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:53.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:53.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:53.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:53.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:53.952 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:53.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:53.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:53.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:53 smithi159 ceph-mon[102519]: pgmap v750: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 3857/282411 objects degraded (1.366%); 5983/282411 objects misplaced (2.119%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:48:54.045 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:48:54.045 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:48:54.046 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-13T14:48:54.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-13T14:48:54.075 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:53 smithi042 ceph-mon[113926]: pgmap v750: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 3857/282411 objects degraded (1.366%); 5983/282411 objects misplaced (2.119%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:48:54.076 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:53 smithi042 ceph-mon[108570]: pgmap v750: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 3857/282411 objects degraded (1.366%); 5983/282411 objects misplaced (2.119%); 5.2 MiB/s, 82 objects/s recovering 2024-09-13T14:48:54.083 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:54.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:54.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:54.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:54.345 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:54.366 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:54.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:54.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:54.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:54.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:54.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:54.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:54.722 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:54.758 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:54.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:54.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:54.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:54.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:54 smithi159 ceph-mon[102519]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:48:55.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:54 smithi042 ceph-mon[113926]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:48:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:54 smithi042 ceph-mon[108570]: osdmap e217: 8 total, 8 up, 7 in 2024-09-13T14:48:55.130 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:55.166 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:55.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:55.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:55.439 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:55.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:55.541 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:55.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:55.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:55.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:55.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:55.963 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:55 smithi159 ceph-mon[102519]: pgmap v752: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 3857/280965 objects degraded (1.373%); 5105/280965 objects misplaced (1.817%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:55.963 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:55 smithi159 ceph-mon[102519]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:48:55.963 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3857/280965 objects degraded (1.373%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:55.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:55.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[108570]: pgmap v752: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 3857/280965 objects degraded (1.373%); 5105/280965 objects misplaced (1.817%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[108570]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3857/280965 objects degraded (1.373%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[113926]: pgmap v752: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 106 active+clean; 5.7 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 3857/280965 objects degraded (1.373%); 5105/280965 objects misplaced (1.817%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[113926]: osdmap e218: 8 total, 8 up, 7 in 2024-09-13T14:48:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3857/280965 objects degraded (1.373%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:48:56.124 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:56.226 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.262 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:56.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:56.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:56.395 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:56.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:56.497 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:56.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:56.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:56.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:56.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:56.769 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:56.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:56.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:56.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:56.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:56.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:57.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:57.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:57.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.102 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:48:57.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:57.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:57.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:48:57.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:57.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:48:57.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:57.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:57.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:57.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:57.864 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:57.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:57.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:57 smithi159 ceph-mon[102519]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 3857/271050 objects degraded (1.423%); 4474/271050 objects misplaced (1.651%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:58.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.011 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:57 smithi042 ceph-mon[108570]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 3857/271050 objects degraded (1.423%); 4474/271050 objects misplaced (1.651%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:58.012 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:57 smithi042 ceph-mon[113926]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.10k op/s; 3857/271050 objects degraded (1.423%); 4474/271050 objects misplaced (1.651%); 5.9 MiB/s, 94 objects/s recovering 2024-09-13T14:48:58.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:58.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:58.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:58.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:48:58.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:48:58.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:48:58.459 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:48:58.561 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:48:58.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:58.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:48:58.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:58.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:58.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:58.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:48:58.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:58.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:58.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:48:58.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:59.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:59.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:59.251 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.425 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:48:59.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:48:59.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.694 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:48:59.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:48:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:48:59.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:48:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:48:59.963 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:48:59.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:48:59.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:48:59 smithi159 ceph-mon[102519]: pgmap v755: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.26k op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:49:00.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:48:59 smithi042 ceph-mon[113926]: pgmap v755: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.26k op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:49:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:48:59 smithi042 ceph-mon[108570]: pgmap v755: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.26k op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 8.0 MiB/s, 127 objects/s recovering 2024-09-13T14:49:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:48:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:48:59] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-13T14:49:00.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:00.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:00.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:00.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.392 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:00.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:00.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:00.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:00.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:00.634 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:00.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.688 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:00.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:00.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:00.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:00.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:00.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:00.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/267057 objects degraded (0.371%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:00.999 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/267057 objects degraded (0.371%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:01.000 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/267057 objects degraded (0.371%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:01.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:01.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:01.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:01.372 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:01.474 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:01.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:01.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:01.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:01.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:01.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:01.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:01.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:01.849 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:01.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:01.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:01.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:01.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:01.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:01 smithi159 ceph-mon[102519]: pgmap v756: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:49:02.000 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:01 smithi042 ceph-mon[108570]: pgmap v756: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:49:02.001 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:01 smithi042 ceph-mon[113926]: pgmap v756: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 991/267057 objects degraded (0.371%); 7073/267057 objects misplaced (2.648%); 6.6 MiB/s, 105 objects/s recovering 2024-09-13T14:49:02.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:02.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:02.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:02.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:49:02.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:02.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:02.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:02.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:02.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:02.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:02.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:02.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:02.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:02.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:02.703 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:49:02.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:02.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:02.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2906345668' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:02.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2906345668' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:02.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:02.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:02.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:02.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:02.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2906345668' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:03.030 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:03.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:03.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:03.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:03.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:03.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:03.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:03.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:03.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:03.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:03.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:03.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:03.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:03.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:03.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:03.946 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:03.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:03 smithi159 ceph-mon[102519]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 991/251862 objects degraded (0.393%); 6703/251862 objects misplaced (2.661%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:49:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:03 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:03.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:03 smithi159 ceph-mon[102519]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:49:04.048 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:04.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:04.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[108570]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 991/251862 objects degraded (0.393%); 6703/251862 objects misplaced (2.661%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:49:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[108570]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:49:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[113926]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 991/251862 objects degraded (0.393%); 6703/251862 objects misplaced (2.661%); 5.3 MiB/s, 85 objects/s recovering 2024-09-13T14:49:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:04.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:04.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:03 smithi042 ceph-mon[113926]: osdmap e219: 8 total, 8 up, 7 in 2024-09-13T14:49:04.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.227 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:04.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:04.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:04.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:04.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:04.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:04.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:04.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:04.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:04.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:04.745 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:04.785 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:04.887 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:04.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:04.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:05.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:05.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:05.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:05.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:05.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:05.337 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:05.352 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:49:05.352 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:49:05.352 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:49:05.352 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:49:05.352 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:49:05.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:05.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:05.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:05.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:05.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:05.794 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:05.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:05.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:05.932 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:05.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:05.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:05 smithi159 ceph-mon[102519]: pgmap v759: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 991/250890 objects degraded (0.395%); 6449/250890 objects misplaced (2.570%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:49:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/250890 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:06.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:06.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:06.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:05 smithi042 ceph-mon[108570]: pgmap v759: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 991/250890 objects degraded (0.395%); 6449/250890 objects misplaced (2.570%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:49:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/250890 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:05 smithi042 ceph-mon[113926]: pgmap v759: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 991/250890 objects degraded (0.395%); 6449/250890 objects misplaced (2.570%); 6.4 MiB/s, 102 objects/s recovering 2024-09-13T14:49:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/250890 objects degraded (0.395%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:06.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:06.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:06.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:06.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:06.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:06.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:06.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:06.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:06.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:06.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:06.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:06.734 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:06.772 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:06.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:06.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:06.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:06.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:49:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:07.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.051 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:07.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:07.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:49:07.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:07.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:49:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:49:07.299 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:49:07.299 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:49:07.299 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:49:07.299 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:49:07.314 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:07.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:07.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:07.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:07.598 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:07.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:07.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:07.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:07.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:07.915 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:07.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:07.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:07.998 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:07 smithi042 ceph-mon[108570]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 991/238989 objects degraded (0.415%); 5999/238989 objects misplaced (2.510%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:49:07.998 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:07 smithi042 ceph-mon[113926]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 991/238989 objects degraded (0.415%); 5999/238989 objects misplaced (2.510%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:49:07.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:07 smithi159 ceph-mon[102519]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 991/238989 objects degraded (0.415%); 5999/238989 objects misplaced (2.510%); 5.8 MiB/s, 92 objects/s recovering 2024-09-13T14:49:08.017 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:08.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:08.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:08.442 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:08.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:08.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:08.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:08.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:08.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:08.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:08.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:08.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:49:08.981 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:49:08.981 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-13T14:49:08.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:08.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:08.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:09.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:09.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:09.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:09.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:09.352 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:09.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:09.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:09.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:09.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:09.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:09.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:09.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:09.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:09.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:09.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:09.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:09.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:09 smithi159 ceph-mon[102519]: pgmap v761: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:10.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:10.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:10.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:09 smithi042 ceph-mon[108570]: pgmap v761: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:09 smithi042 ceph-mon[113926]: pgmap v761: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:09] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-13T14:49:10.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:10.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:10.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:10.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:10.366 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:10.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.798 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:10.899 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:10.936 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:10.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:11.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/234315 objects degraded (0.423%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:11.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/234315 objects degraded (0.423%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/234315 objects degraded (0.423%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:11.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:11.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:11.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:11.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.356 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:11.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:11.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:11.479 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:11.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.613 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:11.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:11.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:11.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:11.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:11 smithi042 ceph-mon[108570]: pgmap v762: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:11 smithi042 ceph-mon[113926]: pgmap v762: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:11.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:11.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:11.987 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:11.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:11 smithi159 ceph-mon[102519]: pgmap v762: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 991/234315 objects degraded (0.423%); 5723/234315 objects misplaced (2.442%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:12.024 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.126 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:12.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:12.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:12.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:12.397 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.431 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:12.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:12.533 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:12.568 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:12.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:12.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:12.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:12.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:12.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:12.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:12.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:12.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:12.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:13.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:13.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:13.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:13.231 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:13.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:13.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:13.369 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:13.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.411 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:13.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:13.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:13.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:13.643 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:13.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:13.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:13.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:13.906 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:13.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:13.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:13.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:14.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:13 smithi042 ceph-mon[108570]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 991/220002 objects degraded (0.450%); 5233/220002 objects misplaced (2.379%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:49:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:13 smithi042 ceph-mon[113926]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 991/220002 objects degraded (0.450%); 5233/220002 objects misplaced (2.379%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:49:14.191 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:14.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:13 smithi159 ceph-mon[102519]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 991/220002 objects degraded (0.450%); 5233/220002 objects misplaced (2.379%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:49:14.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:14.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:14.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:14.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:14.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.628 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:14.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:14.729 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:14.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:14.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:14.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:14.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:14.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:14.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:15.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:15.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:15.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:15.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:15.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:15.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:15.872 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:15.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:15.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:16.001 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:16.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:16.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:15 smithi159 ceph-mon[102519]: pgmap v764: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 991/219093 objects degraded (0.452%); 4909/219093 objects misplaced (2.241%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:49:16.051 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/219093 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:15 smithi042 ceph-mon[108570]: pgmap v764: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 991/219093 objects degraded (0.452%); 4909/219093 objects misplaced (2.241%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:49:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/219093 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:15 smithi042 ceph-mon[113926]: pgmap v764: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 939 op/s; 991/219093 objects degraded (0.452%); 4909/219093 objects misplaced (2.241%); 7.1 MiB/s, 113 objects/s recovering 2024-09-13T14:49:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/219093 objects degraded (0.452%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:16.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:16.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.248 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:16.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:16.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:16.517 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:16.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:16.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:16.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:16.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:16.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:16.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:16.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:16.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:17.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.123 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:17.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:17.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:17.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:17.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:17.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:17.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:17.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:17.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:17.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:17.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:17.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:17.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:17.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:17.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:17.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:17.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:17.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:17 smithi159 ceph-mon[102519]: pgmap v765: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 991/207933 objects degraded (0.477%); 4419/207933 objects misplaced (2.125%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:49:18.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:17 smithi042 ceph-mon[108570]: pgmap v765: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 991/207933 objects degraded (0.477%); 4419/207933 objects misplaced (2.125%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:49:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:17 smithi042 ceph-mon[113926]: pgmap v765: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.19k op/s; 991/207933 objects degraded (0.477%); 4419/207933 objects misplaced (2.125%); 6.8 MiB/s, 109 objects/s recovering 2024-09-13T14:49:18.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.099 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:18.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:18.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:18.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:18.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:18.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:18.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:18.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:18.474 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:18.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:18.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:18.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:18.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:18.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:18.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:18.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:18.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:18.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:18.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:19.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:19.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:19.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:19.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:19.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:19.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:19.466 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:19.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:19.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:19.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:19.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:19.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:19.738 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:19.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:19.886 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:19.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:19.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:19.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:20.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:19 smithi042 ceph-mon[113926]: pgmap v766: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:19 smithi042 ceph-mon[108570]: pgmap v766: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:19] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-13T14:49:20.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:20.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:19 smithi159 ceph-mon[102519]: pgmap v766: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:20.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:20.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:20.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:20.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:20.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:20.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:20.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:20.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:20.733 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:20.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:20.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:20.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:20.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:20.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:21.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.037 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:21.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/203724 objects degraded (0.486%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:21.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/203724 objects degraded (0.486%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:21.138 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:21.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/203724 objects degraded (0.486%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:21.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:21.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:21.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:21.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:21.538 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:21.538 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:21.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:21.604 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:21.706 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:21.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:21.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:21.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:21.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:21.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.014 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:21 smithi042 ceph-mon[108570]: pgmap v767: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:21 smithi042 ceph-mon[113926]: pgmap v767: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:22.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.153 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:21 smithi159 ceph-mon[102519]: pgmap v767: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 4.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 849 op/s; 991/203724 objects degraded (0.486%); 4106/203724 objects misplaced (2.015%); 5.9 MiB/s, 93 objects/s recovering 2024-09-13T14:49:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:22.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:22.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:22.393 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:22.442 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:22.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:22.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:22.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:22.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:22.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:22.811 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3933011574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:22.811 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3933011574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:22.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:22.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.854 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:22.956 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:22.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:22.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:22.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:23.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:23.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:23.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:23.232 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:23.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3933011574' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:23.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:23.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:23.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:23.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:23.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:23.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:23.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:23.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:23.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:23.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:23.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:23.816 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:23.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:23.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:23.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:24.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:24.069 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:23 smithi042 ceph-mon[108570]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 991/189582 objects degraded (0.523%); 3560/189582 objects misplaced (1.878%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:24.069 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:23 smithi042 ceph-mon[113926]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 991/189582 objects degraded (0.523%); 3560/189582 objects misplaced (1.878%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:24.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:24.201 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:24.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:24.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:23 smithi159 ceph-mon[102519]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 991/189582 objects degraded (0.523%); 3560/189582 objects misplaced (1.878%); 7.3 MiB/s, 116 objects/s recovering 2024-09-13T14:49:24.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:24.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:24.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:24.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:24.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:24.605 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:24.606 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:24.607 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:24.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:24.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:24.788 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:24.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:24.924 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:24.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.026 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:25.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:25.193 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:25.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:25.334 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:25.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:25.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:25.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:25.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:25.609 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:25.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:25.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:25.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:25.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:25.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:25.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:25.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:25.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:25.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:26.018 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:26.040 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:26.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:25 smithi042 ceph-mon[108570]: pgmap v769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 991/188505 objects degraded (0.526%); 3174/188505 objects misplaced (1.684%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:49:26.043 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/188505 objects degraded (0.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:26.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:25 smithi042 ceph-mon[113926]: pgmap v769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 991/188505 objects degraded (0.526%); 3174/188505 objects misplaced (1.684%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:49:26.043 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/188505 objects degraded (0.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:26.120 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:26.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:25 smithi159 ceph-mon[102519]: pgmap v769: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 991/188505 objects degraded (0.526%); 3174/188505 objects misplaced (1.684%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:49:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/188505 objects degraded (0.526%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:26.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:26.294 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:26.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:26.396 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.429 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:26.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:26.562 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:26.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:26.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:26.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:26.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:26.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:26.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:26.941 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:49:26.942 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:49:26.943 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:49:26.943 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:49:26.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:26.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:26.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:27.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:27.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:27.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:27.255 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:27.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:27.357 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:27.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:27.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:27.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:27.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:27.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:27.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:27.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:27.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:27.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:27.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:27.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:27.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:28.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:27 smithi042 ceph-mon[113926]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 991/176919 objects degraded (0.560%); 2572/176919 objects misplaced (1.454%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:49:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:27 smithi042 ceph-mon[108570]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 991/176919 objects degraded (0.560%); 2572/176919 objects misplaced (1.454%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:49:28.114 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:28.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:28.215 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:27 smithi159 ceph-mon[102519]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.17k op/s; 991/176919 objects degraded (0.560%); 2572/176919 objects misplaced (1.454%); 7.9 MiB/s, 126 objects/s recovering 2024-09-13T14:49:28.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.380 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:28.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:28.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:28.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:28.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:28.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:28.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:49:28.665 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:28.767 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:28.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:28.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:28.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:28.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:28.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:29.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:29.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:29.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:29.169 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:49:29.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:29.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:29.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:29.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:29.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:29.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:49:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:29.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:29.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:29.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:29.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:29.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:29.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:29.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:29.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:29.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:30.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:30.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:30.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:29] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.51.0" 2024-09-13T14:49:30.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:30.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:30.272 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.301 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:30.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:30.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:30 smithi159 ceph-mon[102519]: pgmap v771: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:49:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:30 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:30 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:30 smithi159 ceph-mon[102519]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:49:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:30 smithi159 ceph-mon[102519]: 10.17 scrub starts 2024-09-13T14:49:30.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:30.561 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:30.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[108570]: pgmap v771: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:49:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[108570]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:49:30.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[108570]: 10.17 scrub starts 2024-09-13T14:49:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[113926]: pgmap v771: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.9 MiB/s, 142 objects/s recovering 2024-09-13T14:49:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:49:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4110862131' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:49:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[113926]: osdmap e220: 8 total, 8 up, 7 in 2024-09-13T14:49:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:30 smithi042 ceph-mon[113926]: 10.17 scrub starts 2024-09-13T14:49:30.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:30.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:30.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:30.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:30.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:30.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:30.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:31.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.092 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:31.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:31.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:31.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:31.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:31.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:31.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:31.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:31.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:31 smithi159 ceph-mon[102519]: 10.17 scrub ok 2024-09-13T14:49:31.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/172644 objects degraded (0.574%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:31.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:31.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:31 smithi042 ceph-mon[108570]: 10.17 scrub ok 2024-09-13T14:49:31.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/172644 objects degraded (0.574%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:31 smithi042 ceph-mon[113926]: 10.17 scrub ok 2024-09-13T14:49:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/172644 objects degraded (0.574%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:31.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:31.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:31.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:31.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:31.790 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:31.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:31.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:31.939 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:31.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:31.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:32.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:32.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:32.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:32.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.212 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:32.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.313 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:32 smithi042 ceph-mon[108570]: pgmap v773: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:49:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:32 smithi042 ceph-mon[113926]: pgmap v773: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:49:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:32.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:32.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.481 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:32.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:32.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:32 smithi159 ceph-mon[102519]: pgmap v773: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 991/172644 objects degraded (0.574%); 2131/172644 objects misplaced (1.234%); 8.8 MiB/s, 140 objects/s recovering 2024-09-13T14:49:32.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:32.618 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:32.720 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:32.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:32.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:32.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:32.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:32.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:32.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:32.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:33.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:33.173 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:33.275 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:33.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:33.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:33.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:33.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:33.551 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:33.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:33.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:33.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:33.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:33.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:33.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:33.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:33.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:33.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:33.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:34.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:34.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:34.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:34.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:34.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:34.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:34.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:34 smithi159 ceph-mon[102519]: pgmap v774: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 991/158565 objects degraded (0.625%); 1509/158565 objects misplaced (0.952%); 9.0 MiB/s, 143 objects/s recovering 2024-09-13T14:49:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:34 smithi159 ceph-mon[102519]: 10.16 scrub starts 2024-09-13T14:49:34.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:34.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:34 smithi042 ceph-mon[113926]: pgmap v774: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 991/158565 objects degraded (0.625%); 1509/158565 objects misplaced (0.952%); 9.0 MiB/s, 143 objects/s recovering 2024-09-13T14:49:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:34 smithi042 ceph-mon[113926]: 10.16 scrub starts 2024-09-13T14:49:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:34 smithi042 ceph-mon[108570]: pgmap v774: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 441 op/s; 991/158565 objects degraded (0.625%); 1509/158565 objects misplaced (0.952%); 9.0 MiB/s, 143 objects/s recovering 2024-09-13T14:49:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:34 smithi042 ceph-mon[108570]: 10.16 scrub starts 2024-09-13T14:49:34.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:34.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:34.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:34.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:34.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:34.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:34.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:35.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:35.089 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:35.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:35.191 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:35.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:35.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:35.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:35.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:35 smithi042 ceph-mon[113926]: 10.19 scrub starts 2024-09-13T14:49:35.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:35 smithi042 ceph-mon[108570]: 10.19 scrub starts 2024-09-13T14:49:35.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:35.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:35.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:35.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:35 smithi159 ceph-mon[102519]: 10.19 scrub starts 2024-09-13T14:49:35.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:35.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:35.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:35.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:35.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:35.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:36.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:36.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:36.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:36.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:36.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:36.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:36.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.463 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:36.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:36 smithi159 ceph-mon[102519]: pgmap v775: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 991/157581 objects degraded (0.629%); 1009/157581 objects misplaced (0.640%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:49:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:36 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 991/157581 objects degraded (0.629%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:36.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:36.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[108570]: pgmap v775: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 991/157581 objects degraded (0.629%); 1009/157581 objects misplaced (0.640%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:49:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 991/157581 objects degraded (0.629%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[113926]: pgmap v775: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 107 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 991/157581 objects degraded (0.629%); 1009/157581 objects misplaced (0.640%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T14:49:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 991/157581 objects degraded (0.629%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-13T14:49:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:36.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:36.700 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:49:36.700 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:49:36.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-13T14:49:36.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:36.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:36.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:36.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:36.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:37.335 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:37.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:37 smithi042 ceph-mon[113926]: 10.11 scrub starts 2024-09-13T14:49:37.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:37 smithi042 ceph-mon[108570]: 10.11 scrub starts 2024-09-13T14:49:37.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:37.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:37.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:37 smithi159 ceph-mon[102519]: 10.11 scrub starts 2024-09-13T14:49:37.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:37.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:37.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:37.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:37.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:37.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:37.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:37.840 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:37.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:37.963 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:37.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:37.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:37.982 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:37.983 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:38.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:38.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:38.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:38.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:38.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:38.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:38.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:38 smithi042 ceph-mon[108570]: pgmap v776: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 991/146784 objects degraded (0.675%); 319/146784 objects misplaced (0.217%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:49:38.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:38 smithi042 ceph-mon[108570]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:49:38.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:38 smithi042 ceph-mon[113926]: pgmap v776: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 991/146784 objects degraded (0.675%); 319/146784 objects misplaced (0.217%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:49:38.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:38 smithi042 ceph-mon[113926]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:49:38.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:38.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.495 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:38.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:38 smithi159 ceph-mon[102519]: pgmap v776: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 991/146784 objects degraded (0.675%); 319/146784 objects misplaced (0.217%); 10 MiB/s, 159 objects/s recovering 2024-09-13T14:49:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:38 smithi159 ceph-mon[102519]: osdmap e221: 8 total, 8 up, 7 in 2024-09-13T14:49:38.597 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:38.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:38.738 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:38.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.776 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:38.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:38.877 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:38.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:38.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:39.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:39.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:39.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:39.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:39.303 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:39.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:39.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:49:39.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:39.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:39.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:39 smithi159 ceph-mon[102519]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:49:39.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:39 smithi042 ceph-mon[113926]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:49:39.500 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:39 smithi042 ceph-mon[108570]: osdmap e222: 8 total, 8 up, 7 in 2024-09-13T14:49:39.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:39.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:39.708 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:39.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:39.807 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:39] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-13T14:49:39.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:39.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:39.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:39.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:39.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:39.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.083 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:49:40.083 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:49:40.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:40.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:40.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:40.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:40.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[113926]: pgmap v779: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 107/142425 objects degraded (0.075%); 11 MiB/s, 282 objects/s recovering 2024-09-13T14:49:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[113926]: 10.10 scrub starts 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[113926]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[113926]: 10.10 scrub ok 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[108570]: pgmap v779: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 107/142425 objects degraded (0.075%); 11 MiB/s, 282 objects/s recovering 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[108570]: 10.10 scrub starts 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[108570]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:49:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:40 smithi042 ceph-mon[108570]: 10.10 scrub ok 2024-09-13T14:49:40.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:40.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:40.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:40 smithi159 ceph-mon[102519]: pgmap v779: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 518 op/s; 107/142425 objects degraded (0.075%); 11 MiB/s, 282 objects/s recovering 2024-09-13T14:49:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:40 smithi159 ceph-mon[102519]: 10.10 scrub starts 2024-09-13T14:49:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:40 smithi159 ceph-mon[102519]: osdmap e223: 8 total, 8 up, 7 in 2024-09-13T14:49:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:40 smithi159 ceph-mon[102519]: 10.10 scrub ok 2024-09-13T14:49:40.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:40.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:40.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:40.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:40.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:40.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:40.931 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:40.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:41.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.070 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:41.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.173 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:41.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:41.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:41.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.350 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:41.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:41.451 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.488 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:41 smithi159 ceph-mon[102519]: 10.16 scrub starts 2024-09-13T14:49:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:41 smithi159 ceph-mon[102519]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:49:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:41 smithi159 ceph-mon[102519]: 10.16 scrub ok 2024-09-13T14:49:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 107/142425 objects degraded (0.075%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:41.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[108570]: 10.16 scrub starts 2024-09-13T14:49:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[108570]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:49:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[108570]: 10.16 scrub ok 2024-09-13T14:49:41.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 107/142425 objects degraded (0.075%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[113926]: 10.16 scrub starts 2024-09-13T14:49:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[113926]: osdmap e224: 8 total, 8 up, 7 in 2024-09-13T14:49:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[113926]: 10.16 scrub ok 2024-09-13T14:49:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 107/142425 objects degraded (0.075%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:49:41.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:41.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:41.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:41.747 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:41.849 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:41.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:41.887 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:41.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:41.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:42.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:42.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:42.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:42.249 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:49:42.249 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:49:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:49:42.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:42.301 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[113926]: pgmap v782: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 107/142425 objects degraded (0.075%); 3.3 MiB/s, 275 objects/s recovering 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[113926]: 10.19 scrub starts 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[113926]: 10.19 scrub ok 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[108570]: pgmap v782: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 107/142425 objects degraded (0.075%); 3.3 MiB/s, 275 objects/s recovering 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[108570]: 10.19 scrub starts 2024-09-13T14:49:42.328 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:42 smithi042 ceph-mon[108570]: 10.19 scrub ok 2024-09-13T14:49:42.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:42.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:42 smithi159 ceph-mon[102519]: pgmap v782: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 2.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 363 op/s; 107/142425 objects degraded (0.075%); 3.3 MiB/s, 275 objects/s recovering 2024-09-13T14:49:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:42 smithi159 ceph-mon[102519]: 10.19 scrub starts 2024-09-13T14:49:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:42 smithi159 ceph-mon[102519]: 10.19 scrub ok 2024-09-13T14:49:42.549 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:42.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:42.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:42.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:42.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:42.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:42.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:42.998 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:42.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:42.999 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:43.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:43.140 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:43.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:43.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:43.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:43.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:43 smithi042 ceph-mon[108570]: 10.11 scrub starts 2024-09-13T14:49:43.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:43 smithi042 ceph-mon[108570]: 10.11 scrub ok 2024-09-13T14:49:43.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:43 smithi042 ceph-mon[113926]: 10.11 scrub starts 2024-09-13T14:49:43.453 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:43 smithi042 ceph-mon[113926]: 10.11 scrub ok 2024-09-13T14:49:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:49:43.491 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:49:43.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 'trim stale osdmaps' 2024-09-13T14:49:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:43 smithi159 ceph-mon[102519]: 10.11 scrub starts 2024-09-13T14:49:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:43 smithi159 ceph-mon[102519]: 10.11 scrub ok 2024-09-13T14:49:43.533 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:43.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:43.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:43.709 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:43.812 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:43.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:43.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:49:43.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:43.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:44.082 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:44.115 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:44.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:44.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.259 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:44.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:44.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:44.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:44.451 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:44 smithi042 ceph-mon[108570]: pgmap v783: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 107/132225 objects degraded (0.081%) 2024-09-13T14:49:44.454 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:44 smithi042 ceph-mon[113926]: pgmap v783: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 107/132225 objects degraded (0.081%) 2024-09-13T14:49:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:44 smithi159 ceph-mon[102519]: pgmap v783: 109 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 2.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 107/132225 objects degraded (0.081%) 2024-09-13T14:49:44.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:44.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:44.636 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:44.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:44.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:44.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:44.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:44.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:44.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:45.035 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:45.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:45.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:45.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:45.307 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:45.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:45.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:49:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:49:45.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:45.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:45.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"} : dispatch 2024-09-13T14:49:45.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:45.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:45.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:45.693 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:45.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:45.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:49:45.770 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:49:45.792 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:49:45.792 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:49:45.793 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:45.797 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:45.811 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:49:45.811 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:49:45.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:45.829 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:49:45.829 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:49:45.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:45.846 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:49:45.846 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:49:45.862 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:49:45.862 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:49:45.878 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:49:45.879 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:49:45.896 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:49:45.896 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:49:45.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:45.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:45.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:45.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:46.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.091 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:46.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:46.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:46.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:46.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:46.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:46.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:46.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:46.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: pgmap v784: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail 2024-09-13T14:49:46.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 107/132225 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:49:46.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: osdmap e225: 8 total, 8 up, 7 in 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: osdmap e226: 8 total, 8 up, 7 in 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/454123962' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: pgmap v784: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 107/132225 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: osdmap e225: 8 total, 8 up, 7 in 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: osdmap e226: 8 total, 8 up, 7 in 2024-09-13T14:49:46.486 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:46 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/454123962' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: pgmap v784: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 2.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 107/132225 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: osdmap e225: 8 total, 8 up, 7 in 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: osdmap e226: 8 total, 8 up, 7 in 2024-09-13T14:49:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:46 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/454123962' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:49:46.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:46.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:46.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:46.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:46.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:46.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:46.837 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:46.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:46.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:46.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:47.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.105 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:47.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:47.248 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:47.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:47.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:47.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.407 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:47.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:47.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:47.547 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:47.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:47 smithi042 ceph-mon[108570]: pgmap v787: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:47 smithi042 ceph-mon[108570]: osdmap e227: 8 total, 8 up, 7 in 2024-09-13T14:49:47.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:47 smithi042 ceph-mon[113926]: pgmap v787: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:47.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:47 smithi042 ceph-mon[113926]: osdmap e227: 8 total, 8 up, 7 in 2024-09-13T14:49:47.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:47.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:47.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:47 smithi159 ceph-mon[102519]: pgmap v787: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:47 smithi159 ceph-mon[102519]: osdmap e227: 8 total, 8 up, 7 in 2024-09-13T14:49:47.790 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:47.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:47.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:47.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:47.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:47.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:48.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:48.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:48.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:48.355 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.391 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:49:48.393 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:48.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:48.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:48.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/610720479' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:48 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/610720479' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:48 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:48.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:48.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:48.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/610720479' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:48 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:49:48.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:48.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:48.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:48.924 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:48.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:49.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:49.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:49.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.220 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:49.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:49.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.361 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:49.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:49.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:49.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[113926]: pgmap v789: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:49:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[113926]: osdmap e228: 8 total, 8 up, 7 in 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[108570]: pgmap v789: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-mon[108570]: osdmap e228: 8 total, 8 up, 7 in 2024-09-13T14:49:49.602 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:49.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:49.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:49.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:49 smithi159 ceph-mon[102519]: pgmap v789: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:49 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:49:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:49 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:49:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:49 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:49:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:49 smithi159 ceph-mon[102519]: osdmap e228: 8 total, 8 up, 7 in 2024-09-13T14:49:49.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:49.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:49:49.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:49.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:49.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:50.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:50.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:49] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-13T14:49:50.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:50.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:50.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:50.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:50.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:50.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.594 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:50.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:50.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:50.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:50.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:50.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:50.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:50.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:50.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:51.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.148 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:51.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.250 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:51.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:51.386 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:51.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:51.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:51.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:51.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:51.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:51.657 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:51.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:51.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:51.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:51.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:51.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:51.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:51.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:51.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:52.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:52.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:52.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[108570]: pgmap v791: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:52.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[113926]: pgmap v791: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:52.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:52.213 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:52.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:51 smithi159 ceph-mon[102519]: pgmap v791: 109 pgs: 1 remapped+peering, 108 active+clean; 2.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail 2024-09-13T14:49:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:49:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:49:52.315 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:52.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:52.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:52.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:52.500 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:52.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:52.602 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:52.635 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:52.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:52.737 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:52.766 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:52.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:52.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:52.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:49:52.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:52.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:52.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:53.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:53.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:52 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:49:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:52 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:49:53.141 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:53.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:52 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"} : dispatch 2024-09-13T14:49:53.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:53.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:53.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:53.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:53.430 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:53.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:53.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:53.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:53.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:53.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:53.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:53.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:53.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:53.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:53.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:54.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:54.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:54.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[108570]: pgmap v792: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 2100/106044 objects misplaced (1.980%) 2024-09-13T14:49:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:49:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[108570]: osdmap e229: 8 total, 8 up, 7 in 2024-09-13T14:49:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[113926]: pgmap v792: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 2100/106044 objects misplaced (1.980%) 2024-09-13T14:49:54.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:49:54.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:53 smithi042 ceph-mon[113926]: osdmap e229: 8 total, 8 up, 7 in 2024-09-13T14:49:54.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:54.176 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:54.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:53 smithi159 ceph-mon[102519]: pgmap v792: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 2100/106044 objects misplaced (1.980%) 2024-09-13T14:49:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:53 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2024-09-13T14:49:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:53 smithi159 ceph-mon[102519]: osdmap e229: 8 total, 8 up, 7 in 2024-09-13T14:49:54.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:54.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:54.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:54.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:54.427 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:54.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:54.529 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:54.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:54.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:54.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:54.673 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:54.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:54.775 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:54.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:54.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:54.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:54.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:54.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:55.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:55.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:54 smithi042 ceph-mon[113926]: osdmap e230: 8 total, 8 up, 7 in 2024-09-13T14:49:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:49:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:54 smithi042 ceph-mon[108570]: osdmap e230: 8 total, 8 up, 7 in 2024-09-13T14:49:55.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:49:55.160 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:55.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:54 smithi159 ceph-mon[102519]: osdmap e230: 8 total, 8 up, 7 in 2024-09-13T14:49:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"} : dispatch 2024-09-13T14:49:55.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:55.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:55.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:49:55.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:49:55.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:55.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:55.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:55.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:55.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:55.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:55.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:55.946 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:55.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:55.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:56.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[108570]: pgmap v795: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 2100/104814 objects misplaced (2.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[108570]: osdmap e231: 8 total, 8 up, 7 in 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[108570]: osdmap e232: 8 total, 8 up, 7 in 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[113926]: pgmap v795: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 2100/104814 objects misplaced (2.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:49:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:49:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[113926]: osdmap e231: 8 total, 8 up, 7 in 2024-09-13T14:49:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:55 smithi042 ceph-mon[113926]: osdmap e232: 8 total, 8 up, 7 in 2024-09-13T14:49:56.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.193 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:55 smithi159 ceph-mon[102519]: pgmap v795: 109 pgs: 1 active+recovering+undersized+remapped, 108 active+clean; 2.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 2100/104814 objects misplaced (2.004%); 1.6 MiB/s, 25 objects/s recovering 2024-09-13T14:49:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2024-09-13T14:49:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:55 smithi159 ceph-mon[102519]: osdmap e231: 8 total, 8 up, 7 in 2024-09-13T14:49:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:55 smithi159 ceph-mon[102519]: osdmap e232: 8 total, 8 up, 7 in 2024-09-13T14:49:56.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:56.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:56.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:56.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:56.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:56.565 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:56.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:56.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:56.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:56.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:56.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:56.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:56.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:56.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:57.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:57.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.161 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.191 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:57.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.431 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:57.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:49:57.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:57.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:49:57.658 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[113926]: pgmap v798: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 1530/100449 objects misplaced (1.523%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:49:57.659 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[113926]: osdmap e233: 8 total, 8 up, 7 in 2024-09-13T14:49:57.659 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[113926]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 229'8202 2024-09-13T14:49:57.660 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[108570]: pgmap v798: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 1530/100449 objects misplaced (1.523%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:49:57.660 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[108570]: osdmap e233: 8 total, 8 up, 7 in 2024-09-13T14:49:57.660 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:57 smithi042 ceph-mon[108570]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 229'8202 2024-09-13T14:49:57.691 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:49:57.700 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:57.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:57 smithi159 ceph-mon[102519]: pgmap v798: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 107 active+clean; 2.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 1530/100449 objects misplaced (1.523%); 4.5 MiB/s, 71 objects/s recovering 2024-09-13T14:49:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:57 smithi159 ceph-mon[102519]: osdmap e233: 8 total, 8 up, 7 in 2024-09-13T14:49:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:57 smithi159 ceph-mon[102519]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 229'8202 2024-09-13T14:49:57.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:57.862 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:57.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:57.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:57.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.001 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:58.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:58.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:58.141 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:58.243 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:49:58.279 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:58.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:49:58.381 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.414 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:58.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.516 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.525 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:58 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:58.525 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:58 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:58.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:58.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.657 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:58.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:58.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:58 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:49:58.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:49:58.824 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:58.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:49:58.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:58.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:58.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.059 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:49:59.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:59.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:49:59.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:59.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:59.364 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:59.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:59.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:49:59.598 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:49:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:49:59 smithi042 ceph-mon[108570]: pgmap v800: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:49:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:49:59 smithi042 ceph-mon[113926]: pgmap v800: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:49:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:59.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:49:59.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:49:59.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:49:59 smithi159 ceph-mon[102519]: pgmap v800: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 468 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.9 MiB/s, 62 objects/s recovering 2024-09-13T14:49:59.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:49:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:49:59.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:49:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:49:59.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:49:59.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:00.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:00.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:49:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:49:59] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-09-13T14:50:00.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:00.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:00.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:00.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:00.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:00.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:00.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:50:00.497 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:50:00.498 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:50:00.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:00.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:00.708 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:00.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:00.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:00.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:00 smithi159 ceph-mon[102519]: Health detail: HEALTH_WARN Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:00 smithi159 ceph-mon[102519]: [WRN] PG_DEGRADED: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:00 smithi159 ceph-mon[102519]: pg 10.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-13T14:50:00.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[108570]: Health detail: HEALTH_WARN Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[108570]: [WRN] PG_DEGRADED: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.788 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[108570]: pg 10.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-13T14:50:00.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[113926]: Health detail: HEALTH_WARN Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.788 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[113926]: [WRN] PG_DEGRADED: Degraded data redundancy: 3835/98406 objects degraded (3.897%), 1 pg degraded 2024-09-13T14:50:00.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:00 smithi042 ceph-mon[113926]: pg 10.16 is active+recovery_wait+undersized+degraded+remapped, acting [1,5] 2024-09-13T14:50:00.848 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:00.864 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:00.884 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:00.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:00.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:00.987 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.025 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:217843 10.2 10:4683f5dd:::benchmark_data_smithi042_191918_object94250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.779840+0000", 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0055245320000000004, 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0055533129999999998, 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.035 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217843 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779840+0000", 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779840+0000", 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779842+0000", 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.395e-06 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779847+0000", 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2900000000000002e-06 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779868+0000", 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1109999999999999e-05 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779875+0000", 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9619999999999996e-06 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.779927+0000", 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2292e-05 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.780012+0000", 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4678999999999993e-05 2024-09-13T14:50:01.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.780113+0000", 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010079899999999999 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782985+0000", 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028723590000000001 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784757+0000", 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001771751 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785348+0000", 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059127699999999997 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45032.0:217840 10.1 e233/192 10:827d08fc:::benchmark_data_smithi042_191918_object94247:head v 233'8380, mlcod=233'8378)", 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.782154+0000", 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0032110229999999999, 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033277329999999998, 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782154+0000", 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782154+0000", 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782154+0000", 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.75e-07 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782159+0000", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2880000000000003e-06 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782168+0000", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2660000000000007e-06 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782177+0000", 2024-09-13T14:50:01.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7430000000000008e-06 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782232+0000", 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5523999999999999e-05 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.782250+0000", 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8428999999999999e-05 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785479+0000", 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032289229999999999 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45032.0:217842 10.0 e233/192 10:05fd45bb:::benchmark_data_smithi042_191918_object94249:head v 233'8446, mlcod=233'8443)", 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.783171+0000", 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0021936239999999999, 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002377381, 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.046 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783171+0000", 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783171+0000", 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783172+0000", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9599999999999998e-07 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783177+0000", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2839999999999997e-06 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783189+0000", 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1877e-05 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.783196+0000", 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8979999999999999e-06 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784174+0000", 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00097780799999999993 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784195+0000", 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1277e-05 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "MOSDPGPush(10.15 233/226 [PushOp(10:ab4ebf93:::benchmark_data_smithi042_191918_object107604:head, version: 206'5123, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:ab4ebf93:::benchmark_data_smithi042_191918_object107604:head@206'5123, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-13T14:50:01.050 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.784572+0000", 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.00079295699999999995, 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001053801, 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784572+0000", 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784572+0000", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784572+0000", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6499999999999999e-07 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784608+0000", 2024-09-13T14:50:01.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5787000000000001e-05 2024-09-13T14:50:01.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784624+0000", 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6288999999999999e-05 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784643+0000", 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8890999999999999e-05 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784747+0000", 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010334 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.784761+0000", 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3801000000000001e-05 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45032.0:217843 10.2 e233/192 ondisk, result = 0)", 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.785259+0000", 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.00010572400000000001, 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00040920099999999999, 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.055 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785259+0000", 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785259+0000", 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785260+0000", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0500000000000003e-07 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785264+0000", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2710000000000003e-06 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785277+0000", 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3089e-05 2024-09-13T14:50:01.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785283+0000", 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0569999999999998e-06 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785337+0000", 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3643999999999997e-05 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785343+0000", 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2149999999999998e-06 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785668+0000", 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00032531999999999998 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:217847 10.14 10:2e41de01:::benchmark_data_smithi042_191918_object94254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e233)", 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:50:00.785580+0000", 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9997845, 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000159598, 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217847 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.060 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785580+0000", 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785580+0000", 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785582+0000", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6160000000000001e-06 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785587+0000", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4909999999999996e-06 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785597+0000", 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8940000000000005e-06 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785605+0000", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7170000000000008e-06 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785645+0000", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9551999999999997e-05 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:50:00.785724+0000", 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8992999999999994e-05 2024-09-13T14:50:01.063 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 6 2024-09-13T14:50:01.064 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:50:01.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:01.305 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:01.407 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.451 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:01.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:01.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:01.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:01 smithi159 ceph-mon[102519]: pgmap v801: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:50:01.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:01 smithi042 ceph-mon[108570]: pgmap v801: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:50:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:01 smithi042 ceph-mon[113926]: pgmap v801: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 106 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 355 op/s; 3835/98406 objects degraded (3.897%); 1530/98406 objects misplaced (1.555%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:50:01.857 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:01.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:01.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:01.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:02.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:02.125 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:02.227 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:02.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:02.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:02.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:02.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:02.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:50:02.532 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:50:02.532 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-13T14:50:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:02.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:02.646 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:02.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.685 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:02.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:02.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:02.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:02.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:02.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:02.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:03.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:03.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:03.205 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.334 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:03.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:03.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:03.636 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:03.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:03.748 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:03 smithi159 ceph-mon[102519]: pgmap v802: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 3835/87552 objects degraded (4.380%); 1867/87552 objects misplaced (2.132%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:50:03.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:03.802 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:03 smithi042 ceph-mon[113926]: pgmap v802: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 3835/87552 objects degraded (4.380%); 1867/87552 objects misplaced (2.132%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:50:03.802 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:03 smithi042 ceph-mon[108570]: pgmap v802: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 761 op/s; 3835/87552 objects degraded (4.380%); 1867/87552 objects misplaced (2.132%); 4.8 MiB/s, 77 objects/s recovering 2024-09-13T14:50:03.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:03.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:03.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:03.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.032 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:04.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:04.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.296 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:04.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:04.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:04.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:04.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:04.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:04.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:04.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:04 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/85626 objects degraded (4.479%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:04.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:04.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:04 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/85626 objects degraded (4.479%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:04 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/85626 objects degraded (4.479%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:04.862 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:04.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:04.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:05.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:05.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:05.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:05.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:05.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:05.404 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:05.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:05.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.538 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:05.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:05.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:05.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:05.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:05 smithi159 ceph-mon[102519]: pgmap v803: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 3835/85626 objects degraded (4.479%); 1867/85626 objects misplaced (2.180%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:50:05.780 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:05.818 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:05 smithi042 ceph-mon[113926]: pgmap v803: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 3835/85626 objects degraded (4.479%); 1867/85626 objects misplaced (2.180%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:50:05.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:05 smithi042 ceph-mon[108570]: pgmap v803: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 3835/85626 objects degraded (4.479%); 1867/85626 objects misplaced (2.180%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:50:05.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:05.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:05.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:05.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:06.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.104 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:06.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:06.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:06.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:06.346 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.387 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.489 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:06.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:06.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:06.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:06.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:06.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:06.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:06.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:06.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:06.936 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:06.936 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:06.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:06.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:06.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:06.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:06.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:07.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:07.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.243 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:07.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:50:07.292 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:07.293 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:07.345 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:07.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:07.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:07.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:07.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:07.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:07.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:07.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:07.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:07.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:07.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:07.927 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:07.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:07.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:08.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:07 smithi042 ceph-mon[113926]: pgmap v804: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 3835/81144 objects degraded (4.726%); 1264/81144 objects misplaced (1.558%); 3.2 MiB/s, 52 objects/s recovering 2024-09-13T14:50:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3739174107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:07 smithi042 ceph-mon[108570]: pgmap v804: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 3835/81144 objects degraded (4.726%); 1264/81144 objects misplaced (1.558%); 3.2 MiB/s, 52 objects/s recovering 2024-09-13T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3739174107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:08.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:50:08.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:07 smithi159 ceph-mon[102519]: pgmap v804: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 3835/81144 objects degraded (4.726%); 1264/81144 objects misplaced (1.558%); 3.2 MiB/s, 52 objects/s recovering 2024-09-13T14:50:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3739174107' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:08.348 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.389 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:08.491 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:08.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:08.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:08.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:08.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:08.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:08.935 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:08.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.074 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:09.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:09.109 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:09.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:09.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:09.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:09.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:09.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:09.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:09.493 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:09.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:09.595 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:09.628 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:09.730 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:09.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:09.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:09.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:09.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:09.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:09.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:10.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:09 smithi042 ceph-mon[113926]: pgmap v805: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:10.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:09 smithi042 ceph-mon[108570]: pgmap v805: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:09] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.51.0" 2024-09-13T14:50:10.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:10.171 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:10.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:09 smithi159 ceph-mon[102519]: pgmap v805: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 807 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:10.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:10.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:10.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:10.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:10.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:10.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:10.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:10.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:10.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:10.877 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:10.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:10.978 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:11.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:11.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/71445 objects degraded (5.368%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:11.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/71445 objects degraded (5.368%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:11.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:50:11.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:11.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:11.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/71445 objects degraded (5.368%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:11.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:11.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:11.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:11.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:11.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:11.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:11.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:11.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:11.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:11.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:12.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:12.065 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:11 smithi042 ceph-mon[113926]: pgmap v806: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:12.065 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:11 smithi042 ceph-mon[108570]: pgmap v806: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:12.102 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:12.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:12.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:12.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:11 smithi159 ceph-mon[102519]: pgmap v806: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 748 op/s; 3835/71445 objects degraded (5.368%); 1264/71445 objects misplaced (1.769%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:50:12.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:12.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:12.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:12.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:12.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:12.617 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:12.647 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:12.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.794 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:12.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:12.934 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:12.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:12.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:12.949 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:50:13.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:13.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:13.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:13.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:13.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:13.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:50:13.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:13.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:13.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:13.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:13.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:13.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:13.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:13.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:13.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:13.762 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:13.864 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:13.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:13.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:13.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:13.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[113926]: pgmap v807: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 3835/58440 objects degraded (6.562%); 654/58440 objects misplaced (1.119%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:50:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[113926]: osdmap e234: 8 total, 8 up, 7 in 2024-09-13T14:50:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[108570]: pgmap v807: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 3835/58440 objects degraded (6.562%); 654/58440 objects misplaced (1.119%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:50:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:13 smithi042 ceph-mon[108570]: osdmap e234: 8 total, 8 up, 7 in 2024-09-13T14:50:14.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.171 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:13 smithi159 ceph-mon[102519]: pgmap v807: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 3835/58440 objects degraded (6.562%); 654/58440 objects misplaced (1.119%); 3.9 MiB/s, 61 objects/s recovering 2024-09-13T14:50:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:13 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/408830261' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:50:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:13 smithi159 ceph-mon[102519]: osdmap e234: 8 total, 8 up, 7 in 2024-09-13T14:50:14.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:14.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:14.439 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:14.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:14.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:14.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:14.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.708 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:14.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:14.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:14.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:14.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:14.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:14.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:14.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:14.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:15.073 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:15.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:15.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:15.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:15.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.255 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:15.357 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:15.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:15.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:15.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:15.534 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:15.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:15.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:15.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:15.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:15.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:15.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:15.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:15 smithi042 ceph-mon[108570]: pgmap v809: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 967 op/s; 3835/56595 objects degraded (6.776%); 654/56595 objects misplaced (1.156%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:50:15.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/56595 objects degraded (6.776%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:15.895 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:15 smithi042 ceph-mon[113926]: pgmap v809: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 967 op/s; 3835/56595 objects degraded (6.776%); 654/56595 objects misplaced (1.156%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:50:15.895 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/56595 objects degraded (6.776%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:15.900 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:15.907 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:15.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:15.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.011 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:50:16.011 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:50:16.012 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:50:16.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:16.108 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:16.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:15 smithi159 ceph-mon[102519]: pgmap v809: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 1.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 967 op/s; 3835/56595 objects degraded (6.776%); 654/56595 objects misplaced (1.156%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:50:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/56595 objects degraded (6.776%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:16.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:16.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:16.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:16.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:16.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:16.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:16.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:16.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:16.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:16.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:16.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:16.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:16.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:16.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:16.910 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:16.948 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:16.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:16.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:17.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:50:17.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:17.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:17.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:17.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:17.343 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:17.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:50:17.439 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:50:17.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-13T14:50:17.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:17.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:17.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:17.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.656 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:17.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:17.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:17.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:17.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:17.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:17.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:17.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:18.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:17 smithi042 ceph-mon[108570]: pgmap v810: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 3835/53928 objects degraded (7.111%); 467/53928 objects misplaced (0.866%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:17 smithi042 ceph-mon[113926]: pgmap v810: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 3835/53928 objects degraded (7.111%); 467/53928 objects misplaced (0.866%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:18.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:18.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:18.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:17 smithi159 ceph-mon[102519]: pgmap v810: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.1 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 906 op/s; 3835/53928 objects degraded (7.111%); 467/53928 objects misplaced (0.866%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:18.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.347 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:18.449 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:18.481 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:18.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:18.583 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:18.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:18.714 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:18.756 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:18.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:18.858 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:18.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:18.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:18.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:18.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:19.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:19.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:19.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:19.272 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:19.304 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:19.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:19.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:19.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:19.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:19.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:19.680 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:19.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.722 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:19.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:19.823 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:19.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:19.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.075 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:19 smithi042 ceph-mon[113926]: pgmap v811: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:19 smithi042 ceph-mon[108570]: pgmap v811: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:19] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.51.0" 2024-09-13T14:50:20.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:19 smithi159 ceph-mon[102519]: pgmap v811: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:20.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:20.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:20.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:20.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:20.741 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:20.843 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:20.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:20.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:20.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:20.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:21.002 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/42189 objects degraded (9.090%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:21.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/42189 objects degraded (9.090%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.104 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:21.145 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:21.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:21.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/42189 objects degraded (9.090%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:21.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:21.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:21.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:21.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:21.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:21.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:21.658 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.701 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:21.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:21.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:21.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:21.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:21.918 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:21.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:22.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:22.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:21 smithi042 ceph-mon[108570]: pgmap v812: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:21 smithi042 ceph-mon[113926]: pgmap v812: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:22.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:22.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:22.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:21 smithi159 ceph-mon[102519]: pgmap v812: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 878 MiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 974 op/s; 3835/42189 objects degraded (9.090%); 467/42189 objects misplaced (1.107%); 1.5 MiB/s, 48 objects/s recovering 2024-09-13T14:50:22.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:22.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:22.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:22.413 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:22.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:22.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:22.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:22.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:22.688 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:22.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:22.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:22.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:22.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:22.866 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:50:22.866 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:50:22.867 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:50:22.867 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:50:22.867 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:50:22.921 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:22.922 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:22.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:22.932 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:22.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:22.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:22.973 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:50:22.973 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:22.974 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3893791352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:22.974 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3893791352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:23.006 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:50:23.006 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:23.033 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:50:23.033 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:23.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:23.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:23.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.248 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3893791352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:50:23.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:23.350 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:23.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:23.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:23.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:23.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:23.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:23.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:23.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.793 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:23.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:23.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:23.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:23.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:23 smithi042 ceph-mon[108570]: pgmap v813: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 603 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 980 op/s; 3835/29001 objects degraded (13.224%); 303/29001 objects misplaced (1.045%); 160 KiB/s, 52 objects/s recovering 2024-09-13T14:50:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:23 smithi042 ceph-mon[113926]: pgmap v813: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 603 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 980 op/s; 3835/29001 objects degraded (13.224%); 303/29001 objects misplaced (1.045%); 160 KiB/s, 52 objects/s recovering 2024-09-13T14:50:24.158 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:23 smithi159 ceph-mon[102519]: pgmap v813: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 603 MiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 980 op/s; 3835/29001 objects degraded (13.224%); 303/29001 objects misplaced (1.045%); 160 KiB/s, 52 objects/s recovering 2024-09-13T14:50:24.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.324 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:24.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:24.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:24.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:24.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:24.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:24.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:24.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:24.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.852 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:24.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:24.984 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:24.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:24.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:25.086 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:25.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:25.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:25.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:25.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:25.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:25.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:25.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:25.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:25.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:25.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:25.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:25.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:25.984 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:25.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:25.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:26.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:25 smithi042 ceph-mon[108570]: pgmap v814: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 566 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 3835/27204 objects degraded (14.097%); 303/27204 objects misplaced (1.114%); 139 KiB/s, 45 objects/s recovering 2024-09-13T14:50:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/27204 objects degraded (14.097%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:25 smithi042 ceph-mon[113926]: pgmap v814: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 566 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 3835/27204 objects degraded (14.097%); 303/27204 objects misplaced (1.114%); 139 KiB/s, 45 objects/s recovering 2024-09-13T14:50:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/27204 objects degraded (14.097%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:26.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:26.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:25 smithi159 ceph-mon[102519]: pgmap v814: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 566 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 3835/27204 objects degraded (14.097%); 303/27204 objects misplaced (1.114%); 139 KiB/s, 45 objects/s recovering 2024-09-13T14:50:26.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/27204 objects degraded (14.097%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:26.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:26.355 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:26.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:26.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:26.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:26.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:26.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:26.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:26.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:26.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:27.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:27.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:27.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:27.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:27.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:27.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:27.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:27.481 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:27.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:27.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:27.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:27.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:27.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:27.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:27.886 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:27.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:27.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:28.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:28.061 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:28.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:27 smithi042 ceph-mon[108570]: pgmap v815: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 511 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 3835/24588 objects degraded (15.597%); 157/24588 objects misplaced (0.639%); 133 KiB/s, 65 objects/s recovering 2024-09-13T14:50:28.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:27 smithi042 ceph-mon[113926]: pgmap v815: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 511 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 3835/24588 objects degraded (15.597%); 157/24588 objects misplaced (0.639%); 133 KiB/s, 65 objects/s recovering 2024-09-13T14:50:28.163 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:28.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:28.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:27 smithi159 ceph-mon[102519]: pgmap v815: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 511 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 888 op/s; 3835/24588 objects degraded (15.597%); 157/24588 objects misplaced (0.639%); 133 KiB/s, 65 objects/s recovering 2024-09-13T14:50:28.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:28.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:28.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:28.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:28.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:28.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:28.577 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:28.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:28.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:28.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:50:28.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.756 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:28.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:28.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:28.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:28.897 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:28.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:28.998 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:29.041 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:29.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:29.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:29.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:29.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:29.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:29.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:29.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:29.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:29.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:29.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:29.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:29.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:29.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:29.856 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:50:29.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:29.927 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:29.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:30.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:30.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:30.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:30.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:29] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.51.0" 2024-09-13T14:50:30.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[113926]: pgmap v816: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 43 objects/s recovering 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3428133392' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[108570]: pgmap v816: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 43 objects/s recovering 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3428133392' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:30.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:29 smithi159 ceph-mon[102519]: pgmap v816: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 43 objects/s recovering 2024-09-13T14:50:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:50:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3428133392' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:50:30.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:30.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:30.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:30.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:30.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:30.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:30.525 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:30.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:30.631 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:30.733 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:30.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:30.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:30.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:30.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:30.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:30.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:31.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:31.050 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[108570]: osdmap e235: 8 total, 8 up, 7 in 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/13116 objects degraded (29.239%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:31.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[113926]: osdmap e235: 8 total, 8 up, 7 in 2024-09-13T14:50:31.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/13116 objects degraded (29.239%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:31.116 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 123590 objects 2024-09-13T14:50:31.116 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :129.007 2024-09-13T14:50:31.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:31.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:30 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:50:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:30 smithi159 ceph-mon[102519]: osdmap e235: 8 total, 8 up, 7 in 2024-09-13T14:50:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/13116 objects degraded (29.239%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:31.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:31.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:31.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:31.399 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:31.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:31.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:31.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:31.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:31.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:31.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:50:31.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:31.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:31.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:31.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:31.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:31.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:32.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:32.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:32.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:32.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:32.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:31 smithi159 ceph-mon[102519]: pgmap v818: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 51 objects/s recovering 2024-09-13T14:50:32.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:32.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:32.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:31 smithi042 ceph-mon[108570]: pgmap v818: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 51 objects/s recovering 2024-09-13T14:50:32.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:31 smithi042 ceph-mon[113926]: pgmap v818: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 272 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 3835/13116 objects degraded (29.239%); 157/13116 objects misplaced (1.197%); 0 B/s, 51 objects/s recovering 2024-09-13T14:50:32.396 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:32.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:32.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:32.543 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:32.645 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:32.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:32.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:32.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:32.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:32.893 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:32.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:32.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.042 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:50:33.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:33.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:33.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:33.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:33.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:33.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:33.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:33.769 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:33.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:33.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:33.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:33.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:33.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:34.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:34.042 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:34.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:34.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:34.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:34.184 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:33 smithi159 ceph-mon[102519]: pgmap v819: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 23 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 3835/1170 objects degraded (327.778%); 2/1170 objects misplaced (0.171%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:34.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:34.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:33 smithi042 ceph-mon[108570]: pgmap v819: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 23 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 3835/1170 objects degraded (327.778%); 2/1170 objects misplaced (0.171%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:33 smithi042 ceph-mon[113926]: pgmap v819: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 23 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 3835/1170 objects degraded (327.778%); 2/1170 objects misplaced (0.171%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:34.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:34.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:34.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:34.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:34.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:34.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:34.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:34.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:34.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:34.895 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:34.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:34.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:34.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:35.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:35.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.415 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:35.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:35.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:35.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:35.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:35.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:35.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:35.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:35.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:35.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.016 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.038 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:36.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:36.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:35 smithi159 ceph-mon[102519]: pgmap v820: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 902 op/s; 3835/108 objects degraded (3550.926%); 2/108 objects misplaced (1.852%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3835/108 objects degraded (3550.926%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:36.262 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:36.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[113926]: pgmap v820: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 902 op/s; 3835/108 objects degraded (3550.926%); 2/108 objects misplaced (1.852%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3835/108 objects degraded (3550.926%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:36.295 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[108570]: pgmap v820: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 902 op/s; 3835/108 objects degraded (3550.926%); 2/108 objects misplaced (1.852%); 0 B/s, 55 objects/s recovering 2024-09-13T14:50:36.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3835/108 objects degraded (3550.926%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T14:50:36.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:50:36.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:36.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:36.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:36.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:36.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:36.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:36.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:36.636 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:36.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:36.761 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:36.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:36.797 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-13T14:50:36.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-13T14:50:36.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:36.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:36.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:36.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:37.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:37.032 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:37.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:37.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:37.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:37.261 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:50:37.261 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:50:37.261 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:50:37.261 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:50:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:50:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:50:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:50:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:50:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:50:37.264 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:50:37.264 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:50:37.264 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:50:37.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:37.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:37.405 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:37.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:37.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:37.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:37.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:37.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.780 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:37.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:37.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:37.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:37.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:37.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:38.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:38.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:38.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:38.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:37 smithi159 ceph-mon[102519]: pgmap v821: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3835/78 objects degraded (4916.667%); 0 B/s, 163 objects/s recovering 2024-09-13T14:50:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:37 smithi159 ceph-mon[102519]: osdmap e236: 8 total, 8 up, 7 in 2024-09-13T14:50:38.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:38.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:37 smithi042 ceph-mon[108570]: pgmap v821: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3835/78 objects degraded (4916.667%); 0 B/s, 163 objects/s recovering 2024-09-13T14:50:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:37 smithi042 ceph-mon[108570]: osdmap e236: 8 total, 8 up, 7 in 2024-09-13T14:50:38.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:37 smithi042 ceph-mon[113926]: pgmap v821: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 816 op/s; 3835/78 objects degraded (4916.667%); 0 B/s, 163 objects/s recovering 2024-09-13T14:50:38.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:37 smithi042 ceph-mon[113926]: osdmap e236: 8 total, 8 up, 7 in 2024-09-13T14:50:38.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.400 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:38.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:38.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:38.524 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:38.626 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:38.647 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:38.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:38.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:38.877 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:38.899 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:38.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:38.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:39.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:39.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:39.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:39.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:39.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:39.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:38 smithi159 ceph-mon[102519]: osdmap e237: 8 total, 8 up, 7 in 2024-09-13T14:50:39.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:39.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:38 smithi042 ceph-mon[108570]: osdmap e237: 8 total, 8 up, 7 in 2024-09-13T14:50:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:38 smithi042 ceph-mon[113926]: osdmap e237: 8 total, 8 up, 7 in 2024-09-13T14:50:39.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:39.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:39.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:39.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:39.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:39.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:39.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:39.865 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:39.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:39.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:39.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:39.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:40.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:39 smithi042 ceph-mon[108570]: pgmap v824: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 258 objects/s recovering 2024-09-13T14:50:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:39 smithi042 ceph-mon[113926]: pgmap v824: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 258 objects/s recovering 2024-09-13T14:50:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:39] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-13T14:50:40.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:40.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:40.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:40.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:39 smithi159 ceph-mon[102519]: pgmap v824: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 258 objects/s recovering 2024-09-13T14:50:40.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:40.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:40.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:40.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:40.484 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:40.507 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:40.609 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:40.630 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:40.732 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:40.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:40.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:40.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:40.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:40.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:41.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:41.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:41.106 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:41.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1031/78 objects degraded (1321.795%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:41.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:41.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:41.305 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1031/78 objects degraded (1321.795%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:41.305 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1031/78 objects degraded (1321.795%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:41.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:41.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:41.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:41.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:41.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.623 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:41.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:41.748 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:41.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:41.974 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:41.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:41.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:41.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:42.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:42.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:42.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.247 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:41 smithi159 ceph-mon[102519]: pgmap v825: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 220 objects/s recovering 2024-09-13T14:50:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:41 smithi042 ceph-mon[108570]: pgmap v825: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 220 objects/s recovering 2024-09-13T14:50:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:41 smithi042 ceph-mon[113926]: pgmap v825: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 106 active+clean; 582 KiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 45 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 220 objects/s recovering 2024-09-13T14:50:42.349 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.370 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:42.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:42.597 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:42.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:42.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:42.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:42.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:42.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:42.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:42.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:43.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:43.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:43.219 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:43.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:43.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.366 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:43.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:43.467 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:43.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:43.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:43.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:43.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:43.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:43.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:43.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:43.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:43.963 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:43.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:43.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:44.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:44.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:43 smithi159 ceph-mon[102519]: pgmap v826: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 255 objects/s recovering 2024-09-13T14:50:44.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:43 smithi042 ceph-mon[108570]: pgmap v826: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 255 objects/s recovering 2024-09-13T14:50:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:43 smithi042 ceph-mon[113926]: pgmap v826: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 1031/78 objects degraded (1321.795%); 0 B/s, 255 objects/s recovering 2024-09-13T14:50:44.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:44.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:44.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:44.483 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:44.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:44.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:44.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:44.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:44.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:44.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:44.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:44.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:44.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:44.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:44.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:44.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:45.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.108 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.210 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:45.235 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:45.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:45.337 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:45.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:45.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:45.460 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:45.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:45.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:45.691 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:50:45.692 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:50:45.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-13T14:50:45.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:45.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:45.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:45.884 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:45.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:45.961 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:45.985 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:46.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:45 smithi042 ceph-mon[113926]: pgmap v827: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 223 objects/s recovering 2024-09-13T14:50:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:45 smithi042 ceph-mon[108570]: pgmap v827: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 223 objects/s recovering 2024-09-13T14:50:46.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:46.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:46.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:45 smithi159 ceph-mon[102519]: pgmap v827: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 223 objects/s recovering 2024-09-13T14:50:46.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:46.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:46.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:46.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:46.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:46.628 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:46.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:46.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:46.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:46.880 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:46.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:46.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:46.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:47.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:47.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:47.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:47.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:47.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:47.224 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:50:47.224 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:50:47.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65369 10.12 10:4b9c46a1:::benchmark_data_smithi042_191918_object65368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.886344313, 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6915967300000001, 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.225 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65369 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:50:47.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6199999999999998e-07 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334305+0000", 2024-09-13T14:50:47.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4281e-05 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334314+0000", 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334322+0000", 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9959999999999993e-06 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334933+0000", 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00061030700000000004 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334939+0000", 2024-09-13T14:50:47.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.601e-06 2024-09-13T14:50:47.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.005813+0000", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.670873692 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.005962+0000", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149417 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.006134+0000", 2024-09-13T14:50:47.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017209199999999999 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010130+0000", 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039954209999999999 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016394+0000", 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062645540000000003 2024-09-13T14:50:47.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.024165+0000", 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0077701740000000004 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025313+0000", 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011481270000000001 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025350+0000", 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7178000000000002e-05 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025877+0000", 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00052737499999999996 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.234 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65380 10.12 10:4b19149e:::benchmark_data_smithi042_191918_object65379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88629283899999, 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.697573869, 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.235 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65380 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334333+0000", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3799999999999997e-07 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334354+0000", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1263999999999998e-05 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334362+0000", 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3629999999999994e-06 2024-09-13T14:50:47.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334368+0000", 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8850000000000004e-06 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334964+0000", 2024-09-13T14:50:47.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059618500000000005 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334971+0000", 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8560000000000001e-06 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008063+0000", 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6730918909999999 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008164+0000", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100981 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008268+0000", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010392500000000001 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010281+0000", 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020129570000000001 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019809+0000", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0095286329999999999 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025983+0000", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006173825 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.030493+0000", 2024-09-13T14:50:47.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045095300000000003 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.030567+0000", 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4090000000000004e-05 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.031906+0000", 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0013389459999999999 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.242 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65390 10.12 10:4d7a0b2a:::benchmark_data_smithi042_191918_object65389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88624735900001, 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6988079219999999, 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65390 2024-09-13T14:50:47.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334378+0000", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3900000000000004e-07 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334403+0000", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5208999999999999e-05 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334411+0000", 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9189999999999999e-06 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334417+0000", 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4240000000000001e-06 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335000+0000", 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00058312299999999995 2024-09-13T14:50:47.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335006+0000", 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6000000000000003e-06 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009095+0000", 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6740885250000002 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009217+0000", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000121959 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009319+0000", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101903 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012251+0000", 2024-09-13T14:50:47.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029319609999999999 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019953+0000", 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0077021370000000004 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.028546+0000", 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.008593481 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.031984+0000", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034381390000000002 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.032052+0000", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7909000000000003e-05 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.033185+0000", 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001132994 2024-09-13T14:50:47.250 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65423 10.12 10:4e8869da:::benchmark_data_smithi042_191918_object65422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88618952300001, 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7009449569999999, 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.251 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65423 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334436+0000", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5200000000000002e-07 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334457+0000", 2024-09-13T14:50:47.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1718000000000001e-05 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334466+0000", 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6430000000000001e-06 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334472+0000", 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0020000000000003e-06 2024-09-13T14:50:47.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335032+0000", 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00055971700000000005 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335038+0000", 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3180000000000003e-06 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010303+0000", 2024-09-13T14:50:47.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6752650099999999 2024-09-13T14:50:47.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010390+0000", 2024-09-13T14:50:47.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6410999999999996e-05 2024-09-13T14:50:47.257 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:47.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010488+0000", 2024-09-13T14:50:47.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8797999999999998e-05 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.013187+0000", 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026985580000000002 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020042+0000", 2024-09-13T14:50:47.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068551489999999996 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.028612+0000", 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0085700380000000003 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034471+0000", 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058586389999999997 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034542+0000", 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1489999999999995e-05 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.035380+0000", 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00083801399999999999 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65445 10.12 10:4aecfbce:::benchmark_data_smithi042_191918_object65444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.262 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88614332399999, 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7099828289999999, 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65445 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-07 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334503+0000", 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1220999999999999e-05 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334512+0000", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5799999999999992e-06 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334518+0000", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2779999999999999e-06 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335060+0000", 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00054184700000000003 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335066+0000", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9699999999999996e-06 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011296+0000", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.676230428 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011444+0000", 2024-09-13T14:50:47.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014804600000000001 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011575+0000", 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000131013 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.268 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.013340+0000", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001764926 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020114+0000", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067741720000000002 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034076+0000", 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013961341 2024-09-13T14:50:47.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.042908+0000", 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088317910000000003 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.270 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.042956+0000", 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8053000000000002e-05 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.271 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.044464+0000", 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015086780000000001 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.272 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65472 10.12 10:4cbbc5fc:::benchmark_data_smithi042_191918_object65471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88609794999999, 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7122895630000001, 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.273 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65472 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:50:47.274 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.275 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334554+0000", 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6183999999999999e-05 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334564+0000", 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0068000000000001e-05 2024-09-13T14:50:47.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334571+0000", 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2049999999999996e-06 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335092+0000", 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00052121999999999997 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335099+0000", 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5629999999999997e-06 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012291+0000", 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6771921509999999 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.278 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012404+0000", 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113379 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012501+0000", 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6994e-05 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.015452+0000", 2024-09-13T14:50:47.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029508910000000002 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025262+0000", 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098095260000000007 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034134+0000", 2024-09-13T14:50:47.280 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088725550000000007 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.045952+0000", 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011818276000000001 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.045993+0000", 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0343000000000003e-05 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046816+0000", 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00082374 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65476 10.12 10:4d513365:::benchmark_data_smithi042_191918_object65475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.88587777500001, 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.712834333, 2024-09-13T14:50:47.282 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65476 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334748+0000", 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5099999999999999e-07 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334802+0000", 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4203999999999998e-05 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334811+0000", 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8689999999999993e-06 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.285 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334818+0000", 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2880000000000004e-06 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335612+0000", 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00079335099999999997 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.286 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335620+0000", 2024-09-13T14:50:47.287 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9740000000000008e-06 2024-09-13T14:50:47.287 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.287 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.287 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.287 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016436+0000", 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6808159040000001 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016630+0000", 2024-09-13T14:50:47.288 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019332099999999999 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016846+0000", 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021587000000000001 2024-09-13T14:50:47.289 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019905+0000", 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030591199999999998 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.290 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.291 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025923+0000", 2024-09-13T14:50:47.291 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060186570000000002 2024-09-13T14:50:47.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.040837+0000", 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014913786 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046901+0000", 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060638510000000003 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046961+0000", 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0019999999999998e-05 2024-09-13T14:50:47.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047581+0000", 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00062036700000000003 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.294 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65492 10.12 10:4de5edea:::benchmark_data_smithi042_191918_object65491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.885775604, 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.713522875, 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.295 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65492 2024-09-13T14:50:47.296 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.297 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.299 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334850+0000", 2024-09-13T14:50:47.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7700000000000004e-07 2024-09-13T14:50:47.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334911+0000", 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.156e-05 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334920+0000", 2024-09-13T14:50:47.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4910000000000005e-06 2024-09-13T14:50:47.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.301 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334926+0000", 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.635e-06 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335649+0000", 2024-09-13T14:50:47.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00072289499999999998 2024-09-13T14:50:47.303 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335657+0000", 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018279+0000", 2024-09-13T14:50:47.304 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6826222849999999 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018444+0000", 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164267 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018648+0000", 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000204761 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020003+0000", 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001354102 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.026048+0000", 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060454590000000004 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047641+0000", 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021593478999999999 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047749+0000", 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107308 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047822+0000", 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2855999999999995e-05 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.048372+0000", 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00055049199999999997 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.308 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124021 10.14 10:2ba3697e:::benchmark_data_smithi042_191918_object428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.767822416, 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1046680210000002, 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124021 2024-09-13T14:50:47.309 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452804+0000", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7010000000000001e-06 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452810+0000", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5620000000000003e-06 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452828+0000", 2024-09-13T14:50:47.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8241999999999999e-05 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452836+0000", 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1459999999999996e-06 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453443+0000", 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060669599999999999 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453453+0000", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0169999999999999e-05 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.376987+0000", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92353413299999998 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377000+0000", 2024-09-13T14:50:47.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3369e-05 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494162+0000", 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.117161158 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494306+0000", 2024-09-13T14:50:47.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014396600000000001 2024-09-13T14:50:47.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.548321+0000", 2024-09-13T14:50:47.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.054015170000000001 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555032+0000", 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006711308 2024-09-13T14:50:47.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.317 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.317 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555051+0000", 2024-09-13T14:50:47.317 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9389e-05 2024-09-13T14:50:47.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.318 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.318 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.318 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555093+0000", 2024-09-13T14:50:47.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1325000000000002e-05 2024-09-13T14:50:47.318 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.319 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.319 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.319 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555161+0000", 2024-09-13T14:50:47.319 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8522999999999994e-05 2024-09-13T14:50:47.319 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557470+0000", 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023091629999999999 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.320 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.321 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.321 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124045 10.14 10:2fb89687:::benchmark_data_smithi042_191918_object452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.321 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.74898379300001, 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0906675360000002, 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.322 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.323 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.323 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.323 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124045 2024-09-13T14:50:47.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.323 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471642+0000", 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3349999999999999e-06 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471647+0000", 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.955e-06 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471659+0000", 2024-09-13T14:50:47.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2055000000000001e-05 2024-09-13T14:50:47.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.328 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.328 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471666+0000", 2024-09-13T14:50:47.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0890000000000002e-06 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471724+0000", 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7751000000000003e-05 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471734+0000", 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3589999999999997e-06 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.330 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377030+0000", 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90529651300000002 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377038+0000", 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8059999999999995e-06 2024-09-13T14:50:47.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552572+0000", 2024-09-13T14:50:47.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1755340999999999 2024-09-13T14:50:47.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552679+0000", 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010687 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.553038+0000", 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000359148 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.559991+0000", 2024-09-13T14:50:47.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069528999999999997 2024-09-13T14:50:47.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560059+0000", 2024-09-13T14:50:47.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7787999999999995e-05 2024-09-13T14:50:47.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560076+0000", 2024-09-13T14:50:47.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7073999999999999e-05 2024-09-13T14:50:47.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560136+0000", 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0300999999999997e-05 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562309+0000", 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021724919999999998 2024-09-13T14:50:47.339 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124123 10.14 10:2f92ce95:::benchmark_data_smithi042_191918_object530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:50:47.340 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.68174593099999, 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0288850589999998, 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.341 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124123 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538880+0000", 2024-09-13T14:50:47.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4339999999999999e-06 2024-09-13T14:50:47.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538886+0000", 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.203e-06 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538900+0000", 2024-09-13T14:50:47.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4361e-05 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538906+0000", 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.19e-06 2024-09-13T14:50:47.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538972+0000", 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5877000000000006e-05 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538985+0000", 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2565000000000001e-05 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377062+0000", 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.83807765400000001 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377069+0000", 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8399999999999997e-06 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557527+0000", 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.180457504 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557691+0000", 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164358 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.558112+0000", 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000421579 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565029+0000", 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006916663 2024-09-13T14:50:47.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565048+0000", 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9318e-05 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565106+0000", 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7472999999999997e-05 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565161+0000", 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4979000000000003e-05 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567764+0000", 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026030609999999998 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.352 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124160 10.14 10:2a1875cd:::benchmark_data_smithi042_191918_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.65174577600001, 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0036828569999998, 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124160 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568881+0000", 2024-09-13T14:50:47.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.649e-06 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568886+0000", 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1680000000000003e-06 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568898+0000", 2024-09-13T14:50:47.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2198000000000001e-05 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568905+0000", 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568973+0000", 2024-09-13T14:50:47.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7587000000000002e-05 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568983+0000", 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0091e-05 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377092+0000", 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.80810927200000005 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377099+0000", 2024-09-13T14:50:47.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.754e-06 2024-09-13T14:50:47.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562371+0000", 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.185272007 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562528+0000", 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015748699999999999 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562968+0000", 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00043982 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.360 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570098+0000", 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0071301250000000002 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570121+0000", 2024-09-13T14:50:47.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2694e-05 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570194+0000", 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2587000000000001e-05 2024-09-13T14:50:47.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570259+0000", 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5026999999999999e-05 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572562+0000", 2024-09-13T14:50:47.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023032870000000002 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124175 10.14 10:2bb4c019:::benchmark_data_smithi042_191918_object582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.364 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.638510185, 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.996287562, 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.365 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124175 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.368 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582116+0000", 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.697e-06 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582122+0000", 2024-09-13T14:50:47.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.31e-06 2024-09-13T14:50:47.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.370 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582135+0000", 2024-09-13T14:50:47.370 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-05 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.625825+0000", 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.04369029 2024-09-13T14:50:47.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.372 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626947+0000", 2024-09-13T14:50:47.372 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011223139999999999 2024-09-13T14:50:47.372 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626958+0000", 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.039e-05 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377122+0000", 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.75016387699999998 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.375 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377128+0000", 2024-09-13T14:50:47.375 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.708e-06 2024-09-13T14:50:47.375 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.375 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.375 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.376 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567848+0000", 2024-09-13T14:50:47.376 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.19071927 2024-09-13T14:50:47.376 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.376 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.376 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568046+0000", 2024-09-13T14:50:47.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000198648 2024-09-13T14:50:47.379 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.380 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568459+0000", 2024-09-13T14:50:47.380 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041277499999999999 2024-09-13T14:50:47.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.380 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.575988+0000", 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075292850000000001 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576003+0000", 2024-09-13T14:50:47.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4636e-05 2024-09-13T14:50:47.382 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.382 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.382 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.384 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576043+0000", 2024-09-13T14:50:47.384 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0141000000000001e-05 2024-09-13T14:50:47.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576115+0000", 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2324999999999996e-05 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578402+0000", 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022867859999999999 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.385 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124203 10.14 10:2df4aadf:::benchmark_data_smithi042_191918_object610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.58889225999999, 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9513996520000001, 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.386 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124203 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631734+0000", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.393e-06 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631739+0000", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8720000000000001e-06 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631751+0000", 2024-09-13T14:50:47.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2089000000000001e-05 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631758+0000", 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5310000000000004e-06 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631820+0000", 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1235999999999994e-05 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631830+0000", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0087e-05 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377151+0000", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.74532109400000002 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377157+0000", 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.567e-06 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572621+0000", 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1954635090000001 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572838+0000", 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000217176 2024-09-13T14:50:47.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.573258+0000", 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041985200000000002 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578468+0000", 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052102149999999998 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578487+0000", 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9037000000000001e-05 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581071+0000", 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025835509999999999 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581132+0000", 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0745000000000002e-05 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583132+0000", 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002000698 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124303 10.14 10:2cafaf1c:::benchmark_data_smithi042_191918_object710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.49803822800001, 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8779041919999999, 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.394 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124303 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.395 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722588+0000", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.159e-06 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722592+0000", 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1330000000000001e-06 2024-09-13T14:50:47.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722602+0000", 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0475000000000001e-05 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722609+0000", 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6880000000000002e-06 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722665+0000", 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5510000000000002e-05 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722675+0000", 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0122000000000001e-05 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377180+0000", 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.65450549800000002 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377187+0000", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7050000000000002e-06 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578628+0000", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2014406769999999 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578774+0000", 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014649800000000001 2024-09-13T14:50:47.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.579165+0000", 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039142000000000002 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.585995+0000", 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068292939999999996 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590672+0000", 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046776760000000004 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597892+0000", 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072192139999999998 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597941+0000", 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9373000000000003e-05 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600491+0000", 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025497499999999999 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124330 10.14 10:2826c96e:::benchmark_data_smithi042_191918_object737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.478698963, 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.867454178, 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.402 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124330 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741928+0000", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9069999999999999e-06 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741934+0000", 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5509999999999996e-06 2024-09-13T14:50:47.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741957+0000", 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2571000000000001e-05 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741966+0000", 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6219999999999998e-06 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742027+0000", 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1544000000000006e-05 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742037+0000", 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0363999999999999e-05 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377210+0000", 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.63517246999999999 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.406 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377217+0000", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9410000000000001e-06 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583202+0000", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2059855310000001 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583348+0000", 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000145131 2024-09-13T14:50:47.407 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583747+0000", 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039933999999999998 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590701+0000", 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069539379999999998 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.408 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590715+0000", 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4312e-05 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606747+0000", 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016031553 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606846+0000", 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9411000000000001e-05 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.409 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609380+0000", 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025339920000000001 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124347 10.14 10:2b51d2b1:::benchmark_data_smithi042_191918_object754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.46120680199999, 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.852510501, 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.410 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124347 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759419+0000", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6000000000000004e-07 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759421+0000", 2024-09-13T14:50:47.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7700000000000002e-06 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759428+0000", 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1199999999999996e-06 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759435+0000", 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.336e-06 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759494+0000", 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9657000000000002e-05 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759505+0000", 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0743e-05 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377240+0000", 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.61773501500000005 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377247+0000", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5420000000000002e-06 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586075+0000", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.208828676 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586227+0000", 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015113099999999999 2024-09-13T14:50:47.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586620+0000", 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039343899999999998 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590721+0000", 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041013100000000004 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590746+0000", 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5072e-05 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609450+0000", 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018703280999999999 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609614+0000", 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164084 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.611929+0000", 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023147649999999999 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124381 10.14 10:2889f54c:::benchmark_data_smithi042_191918_object788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.430568147, 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.831978273, 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.418 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124381 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790060+0000", 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.774e-06 2024-09-13T14:50:47.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790069+0000", 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7450000000000001e-06 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790074+0000", 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5140000000000001e-06 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790150+0000", 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5729000000000006e-05 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790163+0000", 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2855e-05 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377270+0000", 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.58710728099999998 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377276+0000", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3759999999999996e-06 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588559+0000", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2112829700000001 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588744+0000", 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018437400000000001 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.589096+0000", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035260100000000002 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590759+0000", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016620910000000001 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600557+0000", 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097980979999999999 2024-09-13T14:50:47.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619838+0000", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019281598 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619895+0000", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6721000000000001e-05 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622035+0000", 2024-09-13T14:50:47.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002140028 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124511 10.14 10:2e9e45ad:::benchmark_data_smithi042_191918_object918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.293917596, 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6976293679999999, 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.426 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124511 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926709+0000", 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5600000000000001e-06 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926715+0000", 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0560000000000001e-06 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926732+0000", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6949e-05 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926740+0000", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3559999999999993e-06 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926811+0000", 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0926000000000005e-05 2024-09-13T14:50:47.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926822+0000", 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377300+0000", 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.45047793699999999 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377306+0000", 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6189999999999996e-06 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590780+0000", 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.213474116 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590981+0000", 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000200495 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.591320+0000", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00033930899999999999 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597842+0000", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0065215780000000001 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600611+0000", 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027691840000000001 2024-09-13T14:50:47.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622109+0000", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021498397999999998 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622172+0000", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3184999999999999e-05 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624337+0000", 2024-09-13T14:50:47.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021641899999999999 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124519 10.14 10:2894197e:::benchmark_data_smithi042_191918_object926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 144.28483757000001, 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6923704129999999, 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:50:47.434 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124519 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:50:47.435 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935789+0000", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5149999999999999e-06 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935794+0000", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8069999999999999e-06 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935806+0000", 2024-09-13T14:50:47.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721999999999999e-05 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935813+0000", 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8079999999999999e-06 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935890+0000", 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7139999999999997e-05 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935901+0000", 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1178e-05 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377329+0000", 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.44142803000000003 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377336+0000", 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5459999999999997e-06 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593242+0000", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.215905585 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593400+0000", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015875199999999999 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593797+0000", 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039664300000000002 2024-09-13T14:50:47.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600508+0000", 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067110399999999997 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600684+0000", 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017580100000000001 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624404+0000", 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023720393999999999 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624466+0000", 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2217000000000004e-05 2024-09-13T14:50:47.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.628158+0000", 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036912350000000002 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:50:47.443 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:50:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:47.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:47.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:47.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:47.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:47.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:47.695 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:47.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:47.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:47.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:47.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:47.943 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:47.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:47.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:48.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:48.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:48.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.193 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:48.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.215 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:47 smithi042 ceph-mon[113926]: pgmap v828: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 188 objects/s recovering 2024-09-13T14:50:48.216 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:47 smithi042 ceph-mon[108570]: pgmap v828: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 188 objects/s recovering 2024-09-13T14:50:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:47 smithi159 ceph-mon[102519]: pgmap v828: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 1031/78 objects degraded (1321.795%); 0 B/s, 188 objects/s recovering 2024-09-13T14:50:48.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:48.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:48.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:48.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:48.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.565 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:48.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:48.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:48.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:48.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:48.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:48.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:48.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:48.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:49.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:49.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:48 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:48 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:49.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:49.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:49.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:48 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:50:49.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:49.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:49.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:49.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:49.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:49.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:50:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.565 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:49.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:49.593 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:49.593 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:49.603 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:49.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:49.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:49.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:49.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:49.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:49.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:49] "GET /metrics HTTP/1.1" 200 36446 "" "Prometheus/2.51.0" 2024-09-13T14:50:49.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:49.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:49.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:49.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:50.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:49 smithi159 ceph-mon[102519]: pgmap v829: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 304 objects/s recovering 2024-09-13T14:50:50.287 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.310 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:49 smithi042 ceph-mon[108570]: pgmap v829: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 304 objects/s recovering 2024-09-13T14:50:50.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:49 smithi042 ceph-mon[113926]: pgmap v829: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 304 objects/s recovering 2024-09-13T14:50:50.411 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:50.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:50.657 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:50.680 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:50.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:50.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:50.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:50.906 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:50.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:50.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:50.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:50.962 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:50:51.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:51.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:51.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:51.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/893681786' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:50 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:51.310 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/893681786' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/893681786' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:50:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:50:51.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:51.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:51.540 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:51.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:51.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:51.695 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:51.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:51.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:51.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:51.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:51.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:51.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:52.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:52.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[108570]: pgmap v830: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 229 objects/s recovering 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[108570]: osdmap e238: 8 total, 8 up, 7 in 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[113926]: pgmap v830: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 229 objects/s recovering 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:52.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:51 smithi042 ceph-mon[113926]: osdmap e238: 8 total, 8 up, 7 in 2024-09-13T14:50:52.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:52.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:52.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:51 smithi159 ceph-mon[102519]: pgmap v830: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 925/78 objects degraded (1185.897%); 0 B/s, 229 objects/s recovering 2024-09-13T14:50:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:51 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:50:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:51 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:50:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:51 smithi159 ceph-mon[102519]: osdmap e238: 8 total, 8 up, 7 in 2024-09-13T14:50:52.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:52.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:52.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:52.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:52.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:52.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:52.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:52.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.725 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:52.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:52.826 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.854 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:52.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:52.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:52.979 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:52.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:52.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:53.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:52 smithi042 ceph-mon[108570]: osdmap e239: 8 total, 8 up, 7 in 2024-09-13T14:50:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:52 smithi042 ceph-mon[113926]: osdmap e239: 8 total, 8 up, 7 in 2024-09-13T14:50:53.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:53.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:53.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:53.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:53.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:52 smithi159 ceph-mon[102519]: osdmap e239: 8 total, 8 up, 7 in 2024-09-13T14:50:53.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:53.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:53.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:53.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:53.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:53.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:53.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:53.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:53.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:53.963 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:53.986 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:53.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:54.088 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:54.114 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:54.216 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:54.218 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:53 smithi042 ceph-mon[113926]: pgmap v833: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:54.218 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:53 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded) 2024-09-13T14:50:54.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:53 smithi042 ceph-mon[108570]: pgmap v833: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:54.219 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:53 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded) 2024-09-13T14:50:54.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:54.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:53 smithi159 ceph-mon[102519]: pgmap v833: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:53 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 925/78 objects degraded (1185.897%), 1 pg degraded) 2024-09-13T14:50:54.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:54.370 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:54.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:54.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:54.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:54.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:54.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:54.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:54.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:54.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.751 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:54.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:54.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:50:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:54.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:54.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:50:54.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:55.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:55.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:55.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:55.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:55.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:55.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:55.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:55.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:55.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:55.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:55.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:55.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:55.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:55.688 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:50:55.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:55.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:55.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:55.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:55.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:55.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:55.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:56.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:55 smithi042 ceph-mon[108570]: pgmap v834: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:55 smithi042 ceph-mon[113926]: pgmap v834: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:56.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:56.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:56.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:50:56.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:55 smithi159 ceph-mon[102519]: pgmap v834: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 290 objects/s recovering 2024-09-13T14:50:56.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.407 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:56.561 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:56.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:56.663 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:56.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:56.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:56.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:56.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:56.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:56.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:56.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:57.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:57.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:57.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:57.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:50:57.313 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:57.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:50:57.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:57.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:57.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:57.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:57.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.695 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:57.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:57.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:57.820 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:57.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:57.921 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:57.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:57.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:58.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:50:58.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:58.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:50:58.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:57 smithi042 ceph-mon[113926]: pgmap v835: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:50:58.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:57 smithi042 ceph-mon[108570]: pgmap v835: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:50:58.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:58.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:58.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:57 smithi159 ceph-mon[102519]: pgmap v835: 109 pgs: 1 creating+peering, 108 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:50:58.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:58.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:58.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:58.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:58.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:58.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:58.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:58.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:58.826 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:58.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:58.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:58.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:58.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.051 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:50:59.052 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:50:59.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.068 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:50:59.069 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:50:59.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:59.084 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:50:59.085 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:50:59.101 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:50:59.101 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:50:59.117 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:50:59.118 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:no valid command found; 10 closest matches: 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:0 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:1 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:2 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:abort 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:assert 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:bench [] [] [] [] 2024-09-13T14:50:59.125 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs debug_inject_read_zeros 2024-09-13T14:50:59.126 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs files list 2024-09-13T14:50:59.126 INFO:teuthology.orchestra.run.smithi042.stderr:bluefs stats 2024-09-13T14:50:59.126 INFO:teuthology.orchestra.run.smithi042.stderr:bluestore allocator dump block 2024-09-13T14:50:59.126 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T14:50:59.134 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:50:59.134 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:50:59.151 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:50:59.151 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:50:59.168 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:50:59.168 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:50:59.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:59.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:59.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:59.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:59.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"} : dispatch 2024-09-13T14:50:59.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:50:59.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:59.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:50:59.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:50:59.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:50:59.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:50:59.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:50:59.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:50:59.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:50:59.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:59.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:50:59.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:50:59.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:50:59.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:50:59.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:50:59.985 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:50:59] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-13T14:51:00.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:00.073 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.091 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:00.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:00.192 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:00.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:59 smithi159 ceph-mon[102519]: pgmap v836: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:51:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:51:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:59 smithi159 ceph-mon[102519]: osdmap e240: 8 total, 8 up, 7 in 2024-09-13T14:51:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:50:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1845462122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:00.312 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[113926]: pgmap v836: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:51:00.312 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[113926]: osdmap e240: 8 total, 8 up, 7 in 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1845462122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[108570]: pgmap v836: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s, 115 objects/s recovering 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[108570]: osdmap e240: 8 total, 8 up, 7 in 2024-09-13T14:51:00.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:50:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1845462122' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:00.315 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:00.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:00.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:00.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:00.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:00.567 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:00.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:00.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:00.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:00.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:00.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:00.945 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:00.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:00.972 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:00.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:00.984 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_5' removed 2024-09-13T14:51:01.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:01.094 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:01.105 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:01.105 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:01.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:01.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.219 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:01.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:01.243 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[113926]: osdmap e241: 8 total, 8 up, 7 in 2024-09-13T14:51:01.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:51:01.244 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:51:01.244 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[108570]: osdmap e241: 8 total, 8 up, 7 in 2024-09-13T14:51:01.244 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:51:01.244 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:51:01.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:00 smithi159 ceph-mon[102519]: osdmap e241: 8 total, 8 up, 7 in 2024-09-13T14:51:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:51:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2913320037' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:51:01.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:01.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:01.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:01.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:01.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:01.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:01.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:01.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:01.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:01.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:01.840 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:01.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:01.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:01.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:01.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:02.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:51:02.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:01 smithi159 ceph-mon[102519]: pgmap v839: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:01 smithi159 ceph-mon[102519]: osdmap e242: 8 total, 8 up, 7 in 2024-09-13T14:51:02.260 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:01 smithi042 ceph-mon[113926]: pgmap v839: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:02.261 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:01 smithi042 ceph-mon[113926]: osdmap e242: 8 total, 8 up, 7 in 2024-09-13T14:51:02.261 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:01 smithi042 ceph-mon[108570]: pgmap v839: 109 pgs: 109 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:02.261 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:01 smithi042 ceph-mon[108570]: osdmap e242: 8 total, 8 up, 7 in 2024-09-13T14:51:02.316 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:02.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:02.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:02.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:02.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:02.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:02.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:02.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:02.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:02.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:03.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:03.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:03.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:03.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:03.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:03.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:03.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:03.432 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:03.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:03.556 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:03.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:03.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:03.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:03.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:03.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:03.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:03.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:03.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:03.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:04.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:03 smithi042 ceph-mon[108570]: pgmap v841: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:03 smithi042 ceph-mon[113926]: pgmap v841: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:04.179 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-13T14:51:04.180 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-13T14:51:04.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:04.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:04.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:03 smithi159 ceph-mon[102519]: pgmap v841: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:04.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:04.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:04.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:04.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:04.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:04.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:04.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:04.707 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:04.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:04.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:04.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:04.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:04.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:04.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:04.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:04.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:05.003 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:51:05.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:05.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:05.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:05.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2546102236' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:04 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:05.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2546102236' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.315 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:04 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2546102236' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:04 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:05.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:05.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.459 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:05.561 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.584 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:05.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:05.709 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:05.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:05.836 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:05.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:51:05.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:05.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:05.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:05.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:06.061 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:06.087 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: pgmap v842: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: osdmap e243: 8 total, 8 up, 7 in 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: pgmap v842: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: osdmap e243: 8 total, 8 up, 7 in 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: pgmap v842: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: osdmap e243: 8 total, 8 up, 7 in 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:06.310 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:51:06.311 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:51:06.311 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-09-13T14:51:06.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_6 16 2024-09-13T14:51:06.312 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:06.315 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:06.464 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:06.566 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:06.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:06.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:06.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:06.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:06.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:06.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:06.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:06.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:06.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:06.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:07.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:07.092 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:07.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:07.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:07.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:07.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:07.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:07.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:07.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:07.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:07.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:07.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:07.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:07.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:07.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:07.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:07.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:07.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:51:07.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:07.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:07.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:07.945 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:07.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:07.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:08.072 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:08.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:08.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:07 smithi159 ceph-mon[102519]: pgmap v844: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:08.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:08.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:07 smithi042 ceph-mon[113926]: pgmap v844: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:08.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:07 smithi042 ceph-mon[108570]: pgmap v844: 83 pgs: 83 active+clean; 582 KiB data, 4.9 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:08.347 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:08.449 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.475 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:08.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:08.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:08.743 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:08.744 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:08.744 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:08.744 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:08.829 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:08.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:08.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:08.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:08.976 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:08.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:08.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:09.016 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:09.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.102 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:09.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.180 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:51:09.181 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:51:09.181 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-13T14:51:09.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd in 0 2024-09-13T14:51:09.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.248 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.350 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: pgmap v845: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: pgmap v845: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:09.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:09.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:09.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.374 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:09.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: pgmap v845: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:51:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:51:09.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:09.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:09.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:09.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.627 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:09.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:09.729 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:09.751 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:09.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:09.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:09.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:09.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:10.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.022 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:09] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.51.0" 2024-09-13T14:51:10.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:10.130 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:10.232 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:10.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:10.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:10.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:10.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:10.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:10.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:10.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:10.612 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:10.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:10.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:10.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:10.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:10.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:10.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:10.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:10.993 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:11.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.121 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.144 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:11.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:11.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:11.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:11.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:11.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:11.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:11.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:11.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:11.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:11.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.657 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:11.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:11.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:11 smithi159 ceph-mon[102519]: pgmap v846: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:11.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:11.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:11.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:11 smithi042 ceph-mon[108570]: pgmap v846: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:11 smithi042 ceph-mon[113926]: pgmap v846: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:11.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:11.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:11.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:11.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:12.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:12.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:12.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:12.160 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:12.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:12.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:12.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.415 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:12.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:12.516 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:12.543 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:12.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:12.594 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:51:12.645 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:12.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:12.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:12.769 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:12.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:12.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:12.921 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:12.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:13.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:13.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:13.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:13.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:13.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:13.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:51:13.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:13.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:13.336 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:51:13.336 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:51:13.336 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:13.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:13.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.425 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:13.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:13.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:13 smithi042 ceph-mon[108570]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:13.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2775831635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:13.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:13 smithi042 ceph-mon[113926]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:13.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2775831635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:13.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:13.676 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:13 smithi159 ceph-mon[102519]: pgmap v847: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:13.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2775831635' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:13.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:13.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:13.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:13.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:13.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:14.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:14.049 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:14.151 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:14.177 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:14.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:14.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:14.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:14.531 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:14.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:14.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:14.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:14.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:14.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:14.785 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:14.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:14.910 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:14.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:14.938 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:14.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:15.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:15.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:15.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:15.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.190 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.202 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:15.243 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:15.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:15.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:15.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:15.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:15.427 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_6' created 2024-09-13T14:51:15.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:15 smithi042 ceph-mon[113926]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:15.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-13T14:51:15.445 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:15 smithi042 ceph-mon[108570]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:15.445 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-13T14:51:15.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:15.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:15.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:15.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:15.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:15 smithi159 ceph-mon[102519]: pgmap v848: 83 pgs: 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-09-13T14:51:15.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:15.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:15.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:15.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:15.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:15.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:15.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:16.057 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.087 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:16.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.188 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:16.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:16.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:16.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:16.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:16.441 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:16.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:16.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:16 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-13T14:51:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:16 smithi042 ceph-mon[108570]: osdmap e244: 8 total, 8 up, 7 in 2024-09-13T14:51:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:16 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-13T14:51:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:16 smithi042 ceph-mon[113926]: osdmap e244: 8 total, 8 up, 7 in 2024-09-13T14:51:16.697 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:16.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:16.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:16 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/297022238' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-09-13T14:51:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:16 smithi159 ceph-mon[102519]: osdmap e244: 8 total, 8 up, 7 in 2024-09-13T14:51:16.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:16.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:16.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:16.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:16.971 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:16.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:16.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.099 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:17.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-09-13T14:51:17.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:17.234 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:17.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:17.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:17.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:17.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:17.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:17.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[113926]: pgmap v850: 99 pgs: 7 creating+peering, 9 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[113926]: osdmap e245: 8 total, 8 up, 7 in 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[108570]: pgmap v850: 99 pgs: 7 creating+peering, 9 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:51:17.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:17 smithi042 ceph-mon[108570]: osdmap e245: 8 total, 8 up, 7 in 2024-09-13T14:51:17.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.740 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:17 smithi159 ceph-mon[102519]: pgmap v850: 99 pgs: 7 creating+peering, 9 unknown, 83 active+clean; 582 KiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:17 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:51:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:17 smithi159 ceph-mon[102519]: osdmap e245: 8 total, 8 up, 7 in 2024-09-13T14:51:17.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:17.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:17.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:17.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:17.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.096 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:18.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:18.225 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:18.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:18.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:18.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:18.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:18.447 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:51:18.479 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[108570]: osdmap e246: 8 total, 8 up, 7 in 2024-09-13T14:51:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2157025486' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[113926]: osdmap e246: 8 total, 8 up, 7 in 2024-09-13T14:51:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2157025486' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:18 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.603 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:18.629 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:18.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:18 smithi159 ceph-mon[102519]: osdmap e246: 8 total, 8 up, 7 in 2024-09-13T14:51:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:18 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2157025486' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:18 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:51:18.754 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:18.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:18.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:18.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:18.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:18.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:19.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.008 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:19.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:19.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:19.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:19.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:19.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:19.263 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:19.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:19.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:19.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:19.445 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T14:51:19.170+0000 7fcad2a32640 -1 mon.a@0(leader).osd e247 definitely_dead 0 2024-09-13T14:51:19.455 INFO:teuthology.orchestra.run.smithi042.stderr:marked in osd.0. 2024-09-13T14:51:19.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:19.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:19.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:19.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:19.640 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:19.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: pgmap v853: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:19.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:19.728 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: osdmap e247: 8 total, 8 up, 7 in 2024-09-13T14:51:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/357575221' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: pgmap v853: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:19.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: osdmap e247: 8 total, 8 up, 7 in 2024-09-13T14:51:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/357575221' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: pgmap v853: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: osdmap e247: 8 total, 8 up, 7 in 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/357575221' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:19 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T14:51:19.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:19.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:19.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:19.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:19.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:19.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:20.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:19] "GET /metrics HTTP/1.1" 200 34036 "" "Prometheus/2.51.0" 2024-09-13T14:51:20.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:20.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:20.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:20.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:20.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:20.401 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:20.403 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:20.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:20.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:20.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:20.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:20.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:20.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:20.676 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:51:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:20 smithi159 ceph-mon[102519]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:20 smithi159 ceph-mon[102519]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:20.766 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:51:20.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[108570]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:20.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[108570]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:20.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T14:51:20.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[113926]: osdmap e248: 8 total, 8 up, 8 in 2024-09-13T14:51:20.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:20 smithi042 ceph-mon[113926]: osdmap e249: 8 total, 8 up, 8 in 2024-09-13T14:51:20.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:20.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:20.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:20.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:20.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:21.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:21.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:21.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118880 10.18 10:1b793137:::benchmark_data_smithi042_191918_object118879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:51:21.278 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 186.006737023, 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48544058400000001, 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118880 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:51:21.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264756+0000", 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4000000000000001e-07 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264778+0000", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1586e-05 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264786+0000", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0199999999999994e-06 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264791+0000", 2024-09-13T14:51:21.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6300000000000003e-06 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268136+0000", 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033446109999999999 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268289+0000", 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015365800000000001 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268595+0000", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000305569 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.371699+0000", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.103103899 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750047+0000", 2024-09-13T14:51:21.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.37834784300000002 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750062+0000", 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5671e-05 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750144+0000", 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2013000000000002e-05 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.285 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750196+0000", 2024-09-13T14:51:21.285 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1443999999999997e-05 2024-09-13T14:51:21.285 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.285 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.317 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118882 10.5 10:a1c362d7:::benchmark_data_smithi042_191918_object118881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 186.00432388900001, 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48747538299999998, 2024-09-13T14:51:21.318 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.319 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118882 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267170+0000", 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9609999999999999e-06 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267973+0000", 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00080228000000000005 2024-09-13T14:51:21.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267986+0000", 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.296e-05 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267993+0000", 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4150000000000002e-06 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268200+0000", 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000207099 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268293+0000", 2024-09-13T14:51:21.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2695999999999995e-05 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268535+0000", 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024242600000000001 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369281+0000", 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.100745978 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369337+0000", 2024-09-13T14:51:21.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5943999999999998e-05 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754471+0000", 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.38513415699999998 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754583+0000", 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000111717 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754644+0000", 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0749999999999999e-05 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118887 10.5 10:a472458b:::benchmark_data_smithi042_191918_object118886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 186.00032711, 2024-09-13T14:51:21.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48549398199999999, 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118887 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271167+0000", 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6899999999999999e-06 2024-09-13T14:51:21.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271192+0000", 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4703999999999999e-05 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367626+0000", 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.096434612000000003 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367639+0000", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2571e-05 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369370+0000", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0017312440000000001 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.372032+0000", 2024-09-13T14:51:21.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026619460000000001 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749773+0000", 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.37774075400000001 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754657+0000", 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048840469999999999 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754732+0000", 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5648000000000005e-05 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756520+0000", 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001787468 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756617+0000", 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6817000000000005e-05 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756659+0000", 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2481000000000002e-05 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "MOSDPGPush(10.12 210/200 [PushOp(10:4ee8b4b7:::benchmark_data_smithi042_191918_object5800:head, version: 188'366, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:4ee8b4b7:::benchmark_data_smithi042_191918_object5800:head@188'366, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-09-13T14:51:21.337 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.367656+0000", 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.903836235, 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.39416963700000002, 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367656+0000", 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367656+0000", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367657+0000", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.212e-06 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367709+0000", 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1186000000000001e-05 2024-09-13T14:51:21.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367737+0000", 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8719000000000001e-05 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367754+0000", 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6643999999999999e-05 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.759955+0000", 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.39220137199999999 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.759969+0000", 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3441000000000001e-05 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.761826+0000", 2024-09-13T14:51:21.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018570629999999999 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118600 10.13 10:c8f84504:::benchmark_data_smithi042_191918_object118599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.521801184, 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1358033599999999, 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.342 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118600 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:51:21.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749693+0000", 2024-09-13T14:51:21.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.336e-06 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749768+0000", 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5859000000000007e-05 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749780+0000", 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1355000000000001e-05 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749787+0000", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1740000000000003e-06 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753090+0000", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003302868 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753098+0000", 2024-09-13T14:51:21.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5890000000000003e-06 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.875935+0000", 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1228367180000001 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.876100+0000", 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016534899999999999 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.876294+0000", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019339699999999999 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883004+0000", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067103090000000002 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883054+0000", 2024-09-13T14:51:21.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0083999999999997e-05 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883069+0000", 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4635e-05 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883129+0000", 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9762000000000001e-05 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885495+0000", 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023659250000000001 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118676 10.13 10:ca797c6a:::benchmark_data_smithi042_191918_object118675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.52169395999999, 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.142041785, 2024-09-13T14:51:21.350 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118676 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749799+0000", 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3399999999999998e-07 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749869+0000", 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9334999999999999e-05 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749878+0000", 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5810000000000003e-06 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749892+0000", 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3489000000000001e-05 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753120+0000", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032279800000000001 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753126+0000", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9490000000000001e-06 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.880738+0000", 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1276121480000001 2024-09-13T14:51:21.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.880910+0000", 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017218800000000001 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.881100+0000", 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000189607 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885647+0000", 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004547316 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885681+0000", 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3883000000000001e-05 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.889582+0000", 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039010870000000001 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.889640+0000", 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8053999999999997e-05 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.891840+0000", 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002200334 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118830 10.13 10:cb02c879:::benchmark_data_smithi042_191918_object118829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.52158626600001, 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1522871590000001, 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118830 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.358 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749907+0000", 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6499999999999999e-07 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749960+0000", 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2935000000000001e-05 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749968+0000", 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5459999999999992e-06 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749974+0000", 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1630000000000001e-06 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753147+0000", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031729150000000001 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753153+0000", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0689999999999998e-06 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885720+0000", 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1325662890000001 2024-09-13T14:51:21.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885877+0000", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015730400000000001 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.886036+0000", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015868800000000001 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899913+0000", 2024-09-13T14:51:21.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013877633 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899985+0000", 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1768999999999997e-05 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900133+0000", 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000147818 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900205+0000", 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2096999999999998e-05 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902193+0000", 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019882680000000001 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.367 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118868 10.13 10:cd95e6bd:::benchmark_data_smithi042_191918_object118867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.521507082, 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.154318097, 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.368 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118868 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:51:21.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749986+0000", 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7800000000000001e-07 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750035+0000", 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8702999999999998e-05 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750042+0000", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3869999999999997e-06 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750048+0000", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6810000000000003e-06 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753173+0000", 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031257099999999999 2024-09-13T14:51:21.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753179+0000", 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8150000000000002e-06 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887424+0000", 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1342447490000001 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887532+0000", 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108297 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887710+0000", 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017781200000000001 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899925+0000", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012215014 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900056+0000", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013064500000000001 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902264+0000", 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022082170000000002 2024-09-13T14:51:21.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902331+0000", 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6826e-05 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904303+0000", 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019726629999999999 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.375 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118904 10.13 10:cfdaed63:::benchmark_data_smithi042_191918_object118903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.51644236000001, 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.155650617, 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.376 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118904 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755052+0000", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6109999999999999e-06 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755096+0000", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4175000000000003e-05 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755109+0000", 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3227999999999999e-05 2024-09-13T14:51:21.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755117+0000", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3010000000000007e-06 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755168+0000", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1125999999999998e-05 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755175+0000", 2024-09-13T14:51:21.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5699999999999998e-06 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.891904+0000", 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.13672928 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.892011+0000", 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107039 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.892121+0000", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109954 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900067+0000", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079452470000000008 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908305+0000", 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082383319999999993 2024-09-13T14:51:21.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908429+0000", 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012412400000000001 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908511+0000", 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1971000000000005e-05 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.382 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.910701+0000", 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002189659 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119047 10.13 10:cb7567d0:::benchmark_data_smithi042_191918_object119046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.37089285100001, 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0174962750000001, 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.383 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119047 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.384 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900601+0000", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7540000000000001e-06 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900641+0000", 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9452000000000001e-05 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900660+0000", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9264000000000001e-05 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900670+0000", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0199e-05 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900809+0000", 2024-09-13T14:51:21.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013844000000000001 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900820+0000", 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1568e-05 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.893864+0000", 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99304427299999998 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.894029+0000", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016483599999999999 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.894225+0000", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000196181 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900071+0000", 2024-09-13T14:51:21.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058458820000000002 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915505+0000", 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015433293000000001 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915663+0000", 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015855399999999999 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915735+0000", 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1389000000000004e-05 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918096+0000", 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00236119 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119065 10.13 10:ced35f11:::benchmark_data_smithi042_191918_object119064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.35629651599999, 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.005246254, 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119065 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.391 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915198+0000", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7370000000000001e-06 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915220+0000", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2031999999999998e-05 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915235+0000", 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4924e-05 2024-09-13T14:51:21.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915245+0000", 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0465000000000001e-05 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915303+0000", 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7815000000000001e-05 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915311+0000", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5930000000000006e-06 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.895795+0000", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98048344899999995 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.895961+0000", 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016654500000000001 2024-09-13T14:51:21.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.896165+0000", 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020370800000000001 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900076+0000", 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039105750000000003 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915558+0000", 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015482490999999999 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918158+0000", 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025996370000000001 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918283+0000", 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012484900000000001 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920442+0000", 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002159434 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119099 10.13 10:ce67b412:::benchmark_data_smithi042_191918_object119098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.32422659100001, 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98944306000000004, 2024-09-13T14:51:21.398 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119099 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947268+0000", 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8419999999999999e-06 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947297+0000", 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.915e-05 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947312+0000", 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4877e-05 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947320+0000", 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2770000000000005e-06 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947390+0000", 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9997000000000001e-05 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947401+0000", 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0635999999999999e-05 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.897811+0000", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.95041027600000005 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.897976+0000", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000165241 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.898151+0000", 2024-09-13T14:51:21.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017505900000000001 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904317+0000", 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061655779999999997 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915606+0000", 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011289334 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.934323+0000", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018716602999999998 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.934382+0000", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9710000000000003e-05 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936709+0000", 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023264800000000001 2024-09-13T14:51:21.405 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.409 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.409 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.409 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.409 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119117 10.13 10:cfc8a476:::benchmark_data_smithi042_191918_object119116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.310562756, 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97974125099999998, 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119117 2024-09-13T14:51:21.410 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960931+0000", 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-06 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960951+0000", 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0151000000000001e-05 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960962+0000", 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0699e-05 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960967+0000", 2024-09-13T14:51:21.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8810000000000004e-06 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.961022+0000", 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5495000000000003e-05 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.961031+0000", 2024-09-13T14:51:21.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.884e-06 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904350+0000", 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94331945500000003 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904516+0000", 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016536900000000001 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904724+0000", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000208402 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908320+0000", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035956809999999999 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922915+0000", 2024-09-13T14:51:21.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014594817 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936800+0000", 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013885085 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936842+0000", 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2058000000000003e-05 2024-09-13T14:51:21.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940671+0000", 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038289629999999999 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119145 10.13 10:ce15cb28:::benchmark_data_smithi042_191918_object119144:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.276844853, 2024-09-13T14:51:21.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94818519499999998, 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119145 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994649+0000", 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3260000000000001e-06 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994668+0000", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9227000000000001e-05 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994679+0000", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0873000000000001e-05 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994686+0000", 2024-09-13T14:51:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2529999999999998e-06 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994771+0000", 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5223999999999996e-05 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994783+0000", 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.204e-05 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906355+0000", 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.91157189199999999 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906521+0000", 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016598599999999999 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906723+0000", 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020175600000000001 2024-09-13T14:51:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915445+0000", 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0087218490000000003 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922989+0000", 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075438839999999998 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940736+0000", 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017747338000000001 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940803+0000", 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6346000000000005e-05 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942833+0000", 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020302010000000001 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119172 10.13 10:c92e9377:::benchmark_data_smithi042_191918_object119171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.24949443400001, 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92329959699999997, 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119172 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022006+0000", 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4209999999999995e-06 2024-09-13T14:51:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022041+0000", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4915e-05 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022053+0000", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.204e-05 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022069+0000", 2024-09-13T14:51:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5824999999999998e-05 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022126+0000", 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7309000000000003e-05 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022135+0000", 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5490000000000006e-06 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913097+0000", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.89096235999999995 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913266+0000", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000168927 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913463+0000", 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019709 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922930+0000", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.009466805 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923085+0000", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015497500000000001 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942911+0000", 2024-09-13T14:51:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019825559 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942984+0000", 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3442000000000006e-05 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945298+0000", 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023133799999999999 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119218 10.13 10:cf0f2136:::benchmark_data_smithi042_191918_object119217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.20181563200001, 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.87838980200000005, 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119218 2024-09-13T14:51:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0400000000000002e-07 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069694+0000", 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6555e-05 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069699+0000", 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6269999999999997e-06 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069703+0000", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7000000000000002e-06 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069763+0000", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9670999999999998e-05 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069769+0000", 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.993e-06 2024-09-13T14:51:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920503+0000", 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.850734412 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920653+0000", 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015012900000000001 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920809+0000", 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000156012 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.931881+0000", 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.01107209 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945353+0000", 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013471854 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945401+0000", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7459000000000001e-05 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945460+0000", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8893e-05 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.948067+0000", 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026070030000000001 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119242 10.13 10:ced1767e:::benchmark_data_smithi042_191918_object119241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.17176221, 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.85350327299999995, 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119242 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099732+0000", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4020000000000001e-06 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099757+0000", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5341e-05 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099776+0000", 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9567000000000001e-05 2024-09-13T14:51:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099785+0000", 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7600000000000008e-06 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099874+0000", 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9054999999999994e-05 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099885+0000", 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0862e-05 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923144+0000", 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.82325866999999997 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923298+0000", 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153821 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923484+0000", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000186331 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936730+0000", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013246002999999999 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950271+0000", 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013541169 2024-09-13T14:51:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950313+0000", 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1520000000000002e-05 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950370+0000", 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7769000000000002e-05 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953233+0000", 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028630029999999998 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119243 10.13 10:cc7c2522:::benchmark_data_smithi042_191918_object119242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.17159048400001, 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.85619771600000005, 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119243 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:51:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099903+0000", 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7000000000000003e-07 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099927+0000", 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4298999999999999e-05 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099941+0000", 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4433999999999999e-05 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099949+0000", 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7610000000000007e-06 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.100020+0000", 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0569999999999997e-05 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.100031+0000", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1406999999999999e-05 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925483+0000", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.82545171500000003 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925631+0000", 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014806199999999999 2024-09-13T14:51:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925810+0000", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017913599999999999 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936738+0000", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010927377 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953455+0000", 2024-09-13T14:51:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016717868 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953567+0000", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011176299999999999 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953622+0000", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4991999999999998e-05 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.956100+0000", 2024-09-13T14:51:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024774620000000002 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119249 10.13 10:cbddd720:::benchmark_data_smithi042_191918_object119248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 185.16248986100001, 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.86949032100000001, 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119249 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109005+0000", 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1289999999999999e-06 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109031+0000", 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6843999999999999e-05 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109054+0000", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2503000000000001e-05 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109065+0000", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0645000000000001e-05 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109125+0000", 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0517999999999999e-05 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109132+0000", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1879999999999996e-06 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.927711+0000", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.81857863099999995 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.927847+0000", 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013632299999999999 2024-09-13T14:51:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.928012+0000", 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164556 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936745+0000", 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0087328030000000008 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953515+0000", 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016770804 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.976014+0000", 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022498335000000001 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.976077+0000", 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3629999999999999e-05 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978493+0000", 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002415412 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119304 10.13 10:cb790c47:::benchmark_data_smithi042_191918_object119303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.95049750000001, 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.65975735199999996, 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119304 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320996+0000", 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4419999999999999e-06 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321018+0000", 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1437000000000002e-05 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.465 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321032+0000", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3841e-05 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321039+0000", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.126e-06 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321095+0000", 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.596e-05 2024-09-13T14:51:21.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321103+0000", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6600000000000001e-06 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.929709+0000", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.60860574199999995 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.929886+0000", 2024-09-13T14:51:21.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017648699999999999 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.930067+0000", 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000180951 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950222+0000", 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020155395 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.958813+0000", 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0085912220000000008 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978545+0000", 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019732178 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978614+0000", 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9048999999999996e-05 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:21.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.980752+0000", 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021378619999999999 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:21.470 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:51:21.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:21.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:21.573 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:21.596 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:21.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:21.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:21.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:21.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:21.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:51:21.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:21.847 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:21.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:21.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:21.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:21.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:22.073 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:22.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[108570]: pgmap v856: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[108570]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[113926]: pgmap v856: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:22.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:21 smithi042 ceph-mon[113926]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:22.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:22.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:21 smithi159 ceph-mon[102519]: pgmap v856: 99 pgs: 1 creating+activating, 12 creating+peering, 86 active+clean; 582 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:51:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:21 smithi159 ceph-mon[102519]: osdmap e250: 8 total, 8 up, 8 in 2024-09-13T14:51:22.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:22.351 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:22.453 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:22.476 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:22.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.604 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.706 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:22.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:22.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:22.850 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-13T14:51:22.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:22.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:22.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:22.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:22.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:22 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:51:23.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:22 smithi042 ceph-mon[108570]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:23.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:22 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:51:23.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:22 smithi042 ceph-mon[113926]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:23.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.230 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:22 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:51:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:22 smithi159 ceph-mon[102519]: osdmap e251: 8 total, 8 up, 8 in 2024-09-13T14:51:23.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:23.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:23.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:23.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:23.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:23.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:23.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:23.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:51:23.605 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:23.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:23.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:23.732 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:23.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:23.859 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:23.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:23.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:23.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:24.085 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:23 smithi042 ceph-mon[108570]: pgmap v859: 99 pgs: 2 activating+undersized+remapped, 1 remapped+peering, 1 creating+peering, 95 active+clean; 634 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 1 objects/s recovering 2024-09-13T14:51:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:23 smithi042 ceph-mon[108570]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:23 smithi042 ceph-mon[113926]: pgmap v859: 99 pgs: 2 activating+undersized+remapped, 1 remapped+peering, 1 creating+peering, 95 active+clean; 634 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 1 objects/s recovering 2024-09-13T14:51:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:23 smithi042 ceph-mon[113926]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:24.111 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:24.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:24.213 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:24.236 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:23 smithi159 ceph-mon[102519]: pgmap v859: 99 pgs: 2 activating+undersized+remapped, 1 remapped+peering, 1 creating+peering, 95 active+clean; 634 KiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 1 objects/s recovering 2024-09-13T14:51:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:23 smithi159 ceph-mon[102519]: osdmap e252: 8 total, 8 up, 8 in 2024-09-13T14:51:24.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:24.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:24.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:24.463 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:24.489 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:24.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:24.616 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:24.718 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:24.745 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:24.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:24.783 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:24.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:24.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:24.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:24.973 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:24.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:24.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:24.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:24 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:25.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:24 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:51:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:24 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:24 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:51:25.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:25.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:25.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.248 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:24 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:24 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:51:25.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:25.250 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:25.351 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:25.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:25.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:25.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:25.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:25.600 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:25.622 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:25.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:25.747 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:25.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:25.849 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:25.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:25.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:25.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:26.002 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:26.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:25 smithi042 ceph-mon[108570]: pgmap v862: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 37 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 2 objects/s recovering 2024-09-13T14:51:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:25 smithi042 ceph-mon[108570]: 1.0 scrub starts 2024-09-13T14:51:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:25 smithi042 ceph-mon[113926]: pgmap v862: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 37 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 2 objects/s recovering 2024-09-13T14:51:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:25 smithi042 ceph-mon[113926]: 1.0 scrub starts 2024-09-13T14:51:26.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:26.126 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:26.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:26.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:25 smithi159 ceph-mon[102519]: pgmap v862: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 37 KiB/s rd, 110 KiB/s wr, 27 op/s; 79 KiB/s, 2 objects/s recovering 2024-09-13T14:51:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:25 smithi159 ceph-mon[102519]: 1.0 scrub starts 2024-09-13T14:51:26.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:26.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:26.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.380 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:26.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:26.507 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:26.609 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:26.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:26.735 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:26.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.758 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:26.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:26.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:26.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:26.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:26.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:26.936 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:26.937 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:51:26.938 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:51:26.939 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:26.939 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:26.939 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:26.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:27.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:27.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:26 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3873057579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:27.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:26 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3873057579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:27.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:27.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:27.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:26 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3873057579' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:27.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:27.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:27.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:27.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:27.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:27.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.612 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:27.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:27.737 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:27.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:27.850 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:51:27.851 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:51:27.851 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.7342042230927791 2024-09-13T14:51:27.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 0 0.7342042230927791 2024-09-13T14:51:27.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:27.887 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:27.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:27.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:27.974 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:27 smithi042 ceph-mon[113926]: pgmap v863: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 53 KiB/s, 1 objects/s recovering 2024-09-13T14:51:27.974 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:27 smithi042 ceph-mon[108570]: pgmap v863: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 53 KiB/s, 1 objects/s recovering 2024-09-13T14:51:27.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:28.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:28.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:28.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:28.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:27 smithi159 ceph-mon[102519]: pgmap v863: 99 pgs: 2 peering, 2 activating+undersized+remapped, 95 active+clean; 687 KiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 18 op/s; 53 KiB/s, 1 objects/s recovering 2024-09-13T14:51:28.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:28.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:28.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:28.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:28.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:28.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:28.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:28.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:28.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:28.876 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:28.895 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-09-13T14:51:28.901 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:28.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:29.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:29.029 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:29.130 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:29.217 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3560147710' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.217 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3560147710' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3560147710' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:51:29.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:29.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:29.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:29.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:29.406 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:29.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:29.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:29.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:51:29.798 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:29.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:29.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:29.893 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:29] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.51.0" 2024-09-13T14:51:29.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:29.948 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:29.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:30.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:30.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:30.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:30.196 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:30.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:30.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:29 smithi159 ceph-mon[102519]: pgmap v864: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 7 op/s; 23 KiB/s, 0 objects/s recovering 2024-09-13T14:51:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:51:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:29 smithi159 ceph-mon[102519]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:30.279 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[108570]: pgmap v864: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 7 op/s; 23 KiB/s, 0 objects/s recovering 2024-09-13T14:51:30.279 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:51:30.279 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[108570]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:30.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[113926]: pgmap v864: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 7 op/s; 23 KiB/s, 0 objects/s recovering 2024-09-13T14:51:30.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:51:30.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:29 smithi042 ceph-mon[113926]: osdmap e253: 8 total, 8 up, 8 in 2024-09-13T14:51:30.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:30.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:30.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:30.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:30.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:30.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:30.672 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-09-13T14:51:30.676 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:51:30.676 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:51:30.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:30.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:30.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:30.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:51:30.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:51:30.757 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_223320 2024-09-13T14:51:30.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:51:30.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:51:30.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:30.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:30.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:30.947 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:30.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:30.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.048 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:51:31.048 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:51:31.080 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:30 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:30 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:31.111 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:31.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:31.213 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:30 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:31.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:31.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:31.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:31.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:31.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:31.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:31.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:31.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:31.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3431 3415 213.425 213.438 0.00381784 0.0046719 2024-09-13T14:51:31.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:31.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:31.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:51:31.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:31.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:31.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:32.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.080 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[113926]: pgmap v866: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 9.7 KiB/s rd, 29 KiB/s wr, 6 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-13T14:51:32.080 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:51:32.080 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.080 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[113926]: 1.0 scrub starts 2024-09-13T14:51:32.081 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[108570]: pgmap v866: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 9.7 KiB/s rd, 29 KiB/s wr, 6 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-13T14:51:32.081 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:51:32.081 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.081 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:31 smithi042 ceph-mon[108570]: 1.0 scrub starts 2024-09-13T14:51:32.114 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:32.216 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:31 smithi159 ceph-mon[102519]: pgmap v866: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 687 KiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 9.7 KiB/s rd, 29 KiB/s wr, 6 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-13T14:51:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:31 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:51:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:51:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:31 smithi159 ceph-mon[102519]: 1.0 scrub starts 2024-09-13T14:51:32.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:32.315 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:51:32.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:32.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:32.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:32.658 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:32.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 6707 6691 209.078 204.75 0.00443459 0.00477456 2024-09-13T14:51:32.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:32.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:32.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:32.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:32.928 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:32.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:33.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:33.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:33.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:33.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[108570]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:33.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:33.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:32 smithi042 ceph-mon[113926]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:33.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:33.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:33.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:32 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2989373773' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:51:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:32 smithi159 ceph-mon[102519]: osdmap e254: 8 total, 8 up, 8 in 2024-09-13T14:51:33.307 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:33.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:33.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:33.458 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.489 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:33.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:33.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:33.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:33.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:33.758 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 10013 9997 208.254 206.625 0.00295987 0.00479255 2024-09-13T14:51:33.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:33.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:33.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:34.001 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:34.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:34.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:33 smithi159 ceph-mon[102519]: pgmap v868: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 76 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.5 MiB/s wr, 151 op/s 2024-09-13T14:51:34.281 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:34.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:33 smithi042 ceph-mon[113926]: pgmap v868: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 76 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.5 MiB/s wr, 151 op/s 2024-09-13T14:51:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:33 smithi042 ceph-mon[108570]: pgmap v868: 99 pgs: 2 active+clean+scrubbing, 97 active+clean; 76 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 9.5 MiB/s wr, 151 op/s 2024-09-13T14:51:34.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:34.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:34.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:34.556 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:34.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:34.658 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:34.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:34.736 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:34.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:34.759 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 12887 12871 201.021 179.625 0.00231768 0.00496655 2024-09-13T14:51:34.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:34.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:34.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:34.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:35.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:35.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:35.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:35.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:35.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:51:35.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:35.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:35.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:35.561 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:35.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:35.699 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:35.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:35.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 15444 15428 192.779 159.812 0.00586803 0.00518033 2024-09-13T14:51:35.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:35.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:35.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:35.962 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:35.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:35.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:35.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:36.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:36.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[108570]: pgmap v869: 99 pgs: 99 active+clean; 204 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791} : dispatch 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[113926]: pgmap v869: 99 pgs: 99 active+clean; 204 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-09-13T14:51:36.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:36.183 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.183 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791} : dispatch 2024-09-13T14:51:36.183 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:35 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:36.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:36.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:35 smithi159 ceph-mon[102519]: pgmap v869: 99 pgs: 99 active+clean; 204 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 406 op/s 2024-09-13T14:51:36.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:51:36.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:36.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791} : dispatch 2024-09-13T14:51:36.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:35 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:36.262 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:36.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:36.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:36.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:36.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:36.643 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:36.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:36.759 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 18209 18193 189.449 172.812 0.00322506 0.00527246 2024-09-13T14:51:36.781 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.0 to 0.734204 (bbf4) 2024-09-13T14:51:36.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:36.813 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:36.915 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:36.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:36.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:37.064 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:37.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:37.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.190 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:37.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.414 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.439 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.564 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:37.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:37.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 18603 18587 165.905 24.625 0.00150972 0.00524876 2024-09-13T14:51:37.788 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:37.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:37.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:37.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:37.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:37.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:38.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[108570]: pgmap v870: 99 pgs: 99 active+clean; 566 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791}]': finished 2024-09-13T14:51:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[108570]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[113926]: pgmap v870: 99 pgs: 99 active+clean; 566 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791}]': finished 2024-09-13T14:51:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:37 smithi042 ceph-mon[113926]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:38.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:38.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:37 smithi159 ceph-mon[102519]: pgmap v870: 99 pgs: 99 active+clean; 566 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-13T14:51:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4274151080' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.7342042230927791}]': finished 2024-09-13T14:51:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:37 smithi159 ceph-mon[102519]: osdmap e255: 8 total, 8 up, 8 in 2024-09-13T14:51:38.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:38.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:38.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:38.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:38.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:38.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:38.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:38.708 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:38.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 18603 18587 145.169 0 - 0.00524876 2024-09-13T14:51:38.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:38.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:38.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:38.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:38.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:39.089 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:38 smithi042 ceph-mon[108570]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:39.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:38 smithi042 ceph-mon[108570]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:38 smithi042 ceph-mon[113926]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:39.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:38 smithi042 ceph-mon[113926]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:39.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:39.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:39.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:38 smithi159 ceph-mon[102519]: osdmap e256: 8 total, 8 up, 8 in 2024-09-13T14:51:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:38 smithi159 ceph-mon[102519]: osdmap e257: 8 total, 8 up, 8 in 2024-09-13T14:51:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:39.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:39.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.407 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:39.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:39.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:39.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:39.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:39.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.685 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:39.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:39.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 20517 20501 142.33 59.8125 0.0162896 0.00701784 2024-09-13T14:51:39.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:39.798 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:39] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-13T14:51:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:39.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:39.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:39.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:39.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:39.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:40.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:40.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[108570]: pgmap v873: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-09-13T14:51:40.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2396992924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:40.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[108570]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:40.055 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[113926]: pgmap v873: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-09-13T14:51:40.055 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2396992924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:40.055 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:39 smithi042 ceph-mon[113926]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:40.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:40.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:40.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:39 smithi159 ceph-mon[102519]: pgmap v873: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 156 MiB/s wr, 2.49k op/s 2024-09-13T14:51:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2396992924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:39 smithi159 ceph-mon[102519]: osdmap e258: 8 total, 8 up, 8 in 2024-09-13T14:51:40.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:40.347 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:40.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:40.449 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:40.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:40.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:40.624 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:40.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:40.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:40.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:40.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 22608 22592 141.165 130.688 0.00715315 0.00707829 2024-09-13T14:51:40.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:51:40.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:51:40.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:51:40.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:40.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:40.789 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:40.789 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:40.859 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:40.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:40.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:40.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:41.023 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:41.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:41.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:41.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:41.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:41.257 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:41.267 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:41.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:41.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:41.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:41.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:41.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:41.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:41.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:41.691 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:41.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:41.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 24726 24710 140.365 132.375 0.00501241 0.00711818 2024-09-13T14:51:41.793 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:41.835 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:41.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:41.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:41.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:41.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:42.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:41 smithi042 ceph-mon[113926]: pgmap v876: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:51:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:41 smithi042 ceph-mon[113926]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:51:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:41 smithi042 ceph-mon[108570]: pgmap v876: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:51:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:41 smithi042 ceph-mon[108570]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:51:42.107 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:42.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:42.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:42.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:41 smithi159 ceph-mon[102519]: pgmap v876: 99 pgs: 99 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-13T14:51:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:41 smithi159 ceph-mon[102519]: osdmap e259: 8 total, 8 up, 8 in 2024-09-13T14:51:42.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:42.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:42.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:42.488 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:42.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:42.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:42.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:42.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:42.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:42.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:42.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 26637 26621 138.62 119.438 0.00603438 0.00720783 2024-09-13T14:51:42.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:42.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:42.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:42.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:43.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:43.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:42 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:43.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:42 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:43.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:43.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:42 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:43.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:43.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:43.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:43.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:43.437 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:43.570 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:43.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:43.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:43.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 28674 28658 137.749 127.312 0.00518335 0.00725297 2024-09-13T14:51:43.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.846 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:43.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:43.947 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:43.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:43.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:44.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:43 smithi042 ceph-mon[113926]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1059/67353 objects misplaced (1.572%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T14:51:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:43 smithi042 ceph-mon[108570]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1059/67353 objects misplaced (1.572%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T14:51:44.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:44.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:44.212 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:43 smithi159 ceph-mon[102519]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1059/67353 objects misplaced (1.572%); 1.6 MiB/s, 27 objects/s recovering 2024-09-13T14:51:44.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:44.353 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.392 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:44.493 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.521 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:44.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:44.760 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 30613 30597 136.566 121.188 0.00631881 0.00731679 2024-09-13T14:51:44.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:44.895 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:44.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:44.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:44.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:45.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:45.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:45.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:45.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:45.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:45.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:45.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:45.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:45.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 31945 31929 133.011 83.25 0.00162111 0.00751003 2024-09-13T14:51:45.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:45.954 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:45.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:45.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:46.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:45 smithi042 ceph-mon[113926]: pgmap v879: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s; 1059/73977 objects misplaced (1.432%); 1.2 MiB/s, 21 objects/s recovering 2024-09-13T14:51:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:45 smithi042 ceph-mon[108570]: pgmap v879: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s; 1059/73977 objects misplaced (1.432%); 1.2 MiB/s, 21 objects/s recovering 2024-09-13T14:51:46.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:46.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:46.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:46.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:46.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:45 smithi159 ceph-mon[102519]: pgmap v879: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 1.5 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s; 1059/73977 objects misplaced (1.432%); 1.2 MiB/s, 21 objects/s recovering 2024-09-13T14:51:46.345 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:46.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:46.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:46.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:46.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:46.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:46.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:46.649 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:46.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 33626 33610 131.264 105.062 0.00204148 0.0075844 2024-09-13T14:51:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.774 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:46.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:46.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:46.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:46.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:47.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:47.299 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:47.341 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:47.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:47.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:47.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:47.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:47.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:47.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:47.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 35003 34987 128.605 86.0625 0.00985823 0.00777069 2024-09-13T14:51:47.856 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:47.896 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:47.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:47.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:48.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:48.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:47 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:47 smithi042 ceph-mon[108570]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 712/84114 objects misplaced (0.846%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:51:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:47 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:47 smithi042 ceph-mon[113926]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 712/84114 objects misplaced (0.846%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:51:48.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.172 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:48.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:47 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:47 smithi159 ceph-mon[102519]: pgmap v880: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 1.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 712/84114 objects misplaced (0.846%); 3.8 MiB/s, 62 objects/s recovering 2024-09-13T14:51:48.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:48.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:48.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:48.432 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:48.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:48.533 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:48.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:48.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:48.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:48.721 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:48.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:48.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 36923 36907 128.126 120 0.00664942 0.00779947 2024-09-13T14:51:48.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:48.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:48.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:48.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:48.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:48.996 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:48.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:49.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:49.148 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:49.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:49.250 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:49.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:49.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:49.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:49.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:49.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:49.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:49.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:49.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:49.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:49.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 38821 38805 127.626 118.625 0.0102173 0.00783022 2024-09-13T14:51:49.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:49.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:49.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:49.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:49.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:49.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.027 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:49 smithi042 ceph-mon[108570]: pgmap v881: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s; 712/100818 objects misplaced (0.706%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:51:50.028 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:49] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.51.0" 2024-09-13T14:51:50.028 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:49 smithi042 ceph-mon[113926]: pgmap v881: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s; 712/100818 objects misplaced (0.706%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:51:50.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:50.089 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:50.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:50.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:50.207 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:51:50.208 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:51:50.208 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:51:50.208 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:51:50.208 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:51:50.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:50.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:50.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:49 smithi159 ceph-mon[102519]: pgmap v881: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s; 712/100818 objects misplaced (0.706%); 3.4 MiB/s, 55 objects/s recovering 2024-09-13T14:51:50.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:51:50.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:50.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:50.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:50.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:50.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:50.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:50.635 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:50.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:50.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:50.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:51:50.761106+0000 min lat: 0.00116536 max lat: 2.04401 avg lat: 0.00794727 2024-09-13T14:51:50.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:51:50.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 40262 40246 125.747 90.0625 0.00448402 0.00794727 2024-09-13T14:51:50.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:50.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:50.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:50.910 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:50.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:50.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:51.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:51.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:51.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:51.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:51:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:51.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:51:51.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.312 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:51.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.413 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:51.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:51.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:51.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:51.674 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:51.737 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:51.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:51.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 41789 41773 124.304 95.4375 0.016712 0.00803681 2024-09-13T14:51:51.790 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:51.797 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:51.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:51.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:51.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:51.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:51.916 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:51 smithi042 ceph-mon[108570]: pgmap v882: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 712/100818 objects misplaced (0.706%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:51:51.916 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:51 smithi042 ceph-mon[113926]: pgmap v882: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 712/100818 objects misplaced (0.706%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:51:51.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:52.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:52.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:52.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:52.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:51 smithi159 ceph-mon[102519]: pgmap v882: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s; 712/100818 objects misplaced (0.706%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:51:52.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:52.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:52.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:52.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:52.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:52.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:52.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:52.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 43396 43380 123.218 100.438 0.00724287 0.00810996 2024-09-13T14:51:52.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:52.870 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:52.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:52.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:52.998 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:52.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:52.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:52 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:53.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:52 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:53.100 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:53.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:53.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:53.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:53.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:52 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.279 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:53.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:53.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:53.401 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:51:53.402 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:51:53.402 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_6 2024-09-13T14:51:53.402 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:51:53.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:53.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:53.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:53.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:53.643 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:53.685 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:53.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:53.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 45130 45114 122.573 108.375 0.0072336 0.00815366 2024-09-13T14:51:53.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:53.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:53.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:53.861 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:51:53.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:53.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:53.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:53 smithi042 ceph-mon[113926]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 228/119094 objects misplaced (0.191%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:51:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:53 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:53 smithi042 ceph-mon[108570]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 228/119094 objects misplaced (0.191%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:51:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:53 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:54.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:53 smithi159 ceph-mon[102519]: pgmap v883: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 97 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s; 228/119094 objects misplaced (0.191%); 5.1 MiB/s, 82 objects/s recovering 2024-09-13T14:51:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:53 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:51:54.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:54.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:54.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:54.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:54.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:54.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:54.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:54.704 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:54.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:54.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 45479 45463 118.374 21.8125 0.00222251 0.00818953 2024-09-13T14:51:54.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:54.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:54.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:54.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:54.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:54.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:55.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[113926]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:51:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:54 smithi042 ceph-mon[108570]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:51:55.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:55.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:55.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:55.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:54 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:51:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3998399167' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:51:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:54 smithi159 ceph-mon[102519]: osdmap e260: 8 total, 8 up, 8 in 2024-09-13T14:51:55.343 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:55.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:55.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:55.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:55.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:55.617 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:55.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:55.761 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 15 47080 47065 117.644 100.125 0.00455709 0.00849454 2024-09-13T14:51:55.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:51:55.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:51:55.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:55.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:55.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.078 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:56.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:55 smithi042 ceph-mon[108570]: pgmap v885: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 228/125454 objects misplaced (0.182%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:51:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:55 smithi042 ceph-mon[108570]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:51:56.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:55 smithi042 ceph-mon[113926]: pgmap v885: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 228/125454 objects misplaced (0.182%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:51:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:55 smithi042 ceph-mon[113926]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:51:56.180 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119552 9.3 9:cabe5794:::benchmark_data_smithi042_157727_object119551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 501.287972142, 2024-09-13T14:51:56.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0227897889999999, 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119552 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:51:56.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912390+0000", 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.912392+0000", 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.812e-06 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.915992+0000", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036003739999999999 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.916006+0000", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3910000000000001e-05 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.916015+0000", 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4889999999999995e-06 2024-09-13T14:51:56.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.922079+0000", 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006064687 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.922090+0000", 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0404000000000001e-05 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.914897+0000", 2024-09-13T14:51:56.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99280757100000006 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.917354+0000", 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002456429 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.917594+0000", 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023996900000000001 2024-09-13T14:51:56.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.934990+0000", 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017396838000000001 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935003+0000", 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2974e-05 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935089+0000", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5782e-05 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935139+0000", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9523e-05 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935180+0000", 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1026999999999999e-05 2024-09-13T14:51:56.213 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.34797.0:119593 9.3 9:cfa3dde9:::benchmark_data_smithi042_157727_object119592:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 501.23746997500001, 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97686230799999996, 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.214 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.34797", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119593 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962892+0000", 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962894+0000", 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.731e-06 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962919+0000", 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4768000000000001e-05 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962937+0000", 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8793000000000001e-05 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.962945+0000", 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6210000000000004e-06 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.963030+0000", 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4834999999999997e-05 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:34.963040+0000", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0118e-05 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.927762+0000", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96472230299999995 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.927904+0000", 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014198300000000001 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.928063+0000", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015882299999999999 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935052+0000", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069884939999999996 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.935225+0000", 2024-09-13T14:51:56.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017373900000000001 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939647+0000", 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044221110000000003 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939703+0000", 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5618000000000002e-05 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:43:35.939754+0000", 2024-09-13T14:51:56.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1371e-05 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213820 10.17 10:ee44b033:::benchmark_data_smithi042_191918_object90227:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.82841956, 2024-09-13T14:51:56.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.784238448, 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213820 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:51:56.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371944+0000", 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.595e-06 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.225 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371950+0000", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.716e-06 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371966+0000", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5879000000000002e-05 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371976+0000", 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0655e-05 2024-09-13T14:51:56.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372653+0000", 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00067709599999999997 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372666+0000", 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2221e-05 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.143834+0000", 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.771168766 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.143978+0000", 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014373700000000001 2024-09-13T14:51:56.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.144467+0000", 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00048926199999999997 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.155915+0000", 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011447327 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.155986+0000", 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1088999999999996e-05 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156039+0000", 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2808e-05 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156052+0000", 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3793e-05 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156102+0000", 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9953000000000001e-05 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156181+0000", 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8551000000000001e-05 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.231 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213886 10.17 10:eef5bd04:::benchmark_data_smithi042_191918_object90293:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828371215, 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7933409569999998, 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213886 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-07 2024-09-13T14:51:56.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.234 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.234 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371994+0000", 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4619999999999999e-06 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372001+0000", 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5830000000000001e-06 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372008+0000", 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.156e-06 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372698+0000", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000690615 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372703+0000", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4680000000000003e-06 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156224+0000", 2024-09-13T14:51:56.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7835214420000001 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156310+0000", 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5722000000000004e-05 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156714+0000", 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000404127 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161136+0000", 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044215519999999996 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161150+0000", 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4524e-05 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165147+0000", 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039972719999999996 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165210+0000", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2856999999999998e-05 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165247+0000", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6650999999999999e-05 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165332+0000", 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5041000000000001e-05 2024-09-13T14:51:56.239 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213947 10.17 10:ebf6dfbb:::benchmark_data_smithi042_191918_object90354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.82834717, 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.793580567, 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.240 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213947 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:51:56.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0900000000000002e-07 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372017+0000", 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.052e-06 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372024+0000", 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2639999999999997e-06 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372029+0000", 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1309999999999997e-06 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372732+0000", 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00070308600000000001 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372738+0000", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2230000000000004e-06 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.158652+0000", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7859140469999999 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.158784+0000", 2024-09-13T14:51:56.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132275 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.159222+0000", 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000437637 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161209+0000", 2024-09-13T14:51:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001987366 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165086+0000", 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003876586 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165456+0000", 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00037029700000000001 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165487+0000", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0728000000000001e-05 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165518+0000", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0593000000000003e-05 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165596+0000", 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7873000000000005e-05 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.247 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213949 10.17 10:eaeba952:::benchmark_data_smithi042_191918_object90356:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828327593, 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.799407583, 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.248 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213949 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372037+0000", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6500000000000001e-06 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372047+0000", 2024-09-13T14:51:56.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0339999999999999e-05 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372052+0000", 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1590000000000001e-06 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372770+0000", 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00071784300000000002 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372776+0000", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3539999999999996e-06 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161242+0000", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7884659549999999 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161355+0000", 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011312299999999999 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.253 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:56.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:56.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:55 smithi159 ceph-mon[102519]: pgmap v885: 99 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 97 active+clean; 2.6 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 228/125454 objects misplaced (0.182%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:51:56.254 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:55 smithi159 ceph-mon[102519]: osdmap e261: 8 total, 8 up, 8 in 2024-09-13T14:51:56.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161609+0000", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025334000000000001 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165384+0000", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037758560000000002 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165402+0000", 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7918999999999999e-05 2024-09-13T14:51:56.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169316+0000", 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039136450000000003 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171256+0000", 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019401290000000001 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171320+0000", 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3584000000000007e-05 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171442+0000", 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122323 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213960 10.17 10:ede664c3:::benchmark_data_smithi042_191918_object90367:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:51:56.257 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.82830454, 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.807093547, 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213960 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4400000000000001e-07 2024-09-13T14:51:56.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372060+0000", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.685e-06 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372066+0000", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1709999999999999e-06 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372071+0000", 2024-09-13T14:51:56.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8960000000000002e-06 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372811+0000", 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00074009900000000003 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372819+0000", 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1869999999999997e-06 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163146+0000", 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7903273820000001 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163218+0000", 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1724000000000006e-05 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163517+0000", 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00029936699999999998 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165607+0000", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020893909999999999 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165647+0000", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0482000000000003e-05 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171492+0000", 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058448980000000003 2024-09-13T14:51:56.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179017+0000", 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075249970000000003 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179067+0000", 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9592000000000002e-05 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179151+0000", 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4332000000000005e-05 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213988 10.17 10:edab4be2:::benchmark_data_smithi042_191918_object90395:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.82828442900001, 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8073287489999998, 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.265 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213988 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3500000000000002e-07 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372080+0000", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7999999999999999e-06 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372085+0000", 2024-09-13T14:51:56.267 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4609999999999999e-06 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372089+0000", 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0600000000000001e-06 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372847+0000", 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00075744500000000004 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.268 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372853+0000", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8830000000000003e-06 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165685+0000", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7928323150000001 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165756+0000", 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0901000000000004e-05 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.269 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.166010+0000", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000253945 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169332+0000", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033222019999999998 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171543+0000", 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022114330000000001 2024-09-13T14:51:56.270 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179213+0000", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076700029999999999 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179264+0000", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0346000000000002e-05 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179298+0000", 2024-09-13T14:51:56.271 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3772999999999997e-05 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179406+0000", 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108847 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213999 10.17 10:e927b90b:::benchmark_data_smithi042_191918_object90406:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:51:56.272 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828267662, 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.812156871, 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213999 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:51:56.273 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372095+0000", 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9900000000000002e-07 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.274 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372096+0000", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.544e-06 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372102+0000", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2630000000000003e-06 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372106+0000", 2024-09-13T14:51:56.275 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4719999999999997e-06 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372911+0000", 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00080517800000000004 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372919+0000", 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3159999999999999e-06 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.276 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167500+0000", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7945812290000001 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167599+0000", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9600999999999997e-05 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167791+0000", 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000191218 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.277 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171561+0000", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003770027 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179472+0000", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007911019 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.183988+0000", 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045164380000000002 2024-09-13T14:51:56.278 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184143+0000", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015473900000000001 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184181+0000", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7799000000000002e-05 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184251+0000", 2024-09-13T14:51:56.279 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0729000000000007e-05 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214016 10.17 10:ec50e907:::benchmark_data_smithi042_191918_object90423:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.82824868900001, 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.833510296, 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.280 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214016 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:51:56.281 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372114+0000", 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6600000000000002e-07 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372116+0000", 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.782e-06 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.282 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372122+0000", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4150000000000004e-06 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372127+0000", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6149999999999999e-06 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372940+0000", 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00081335999999999995 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.283 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372946+0000", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1399999999999997e-06 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169379+0000", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7964327500000001 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169460+0000", 2024-09-13T14:51:56.284 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1364000000000002e-05 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169676+0000", 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021588999999999999 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171567+0000", 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018909560000000001 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.285 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184041+0000", 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012474071 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184100+0000", 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9222999999999999e-05 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205461+0000", 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021360494000000001 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.286 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205515+0000", 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.38e-05 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205624+0000", 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010907 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.287 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214058 10.17 10:eb527693:::benchmark_data_smithi042_191918_object90465:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828230276, 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8338334629999999, 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214058 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.288 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:51:56.289 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372134+0000", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372141+0000", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.063e-06 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372146+0000", 2024-09-13T14:51:56.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5310000000000003e-06 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372996+0000", 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085073799999999997 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373003+0000", 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9179999999999997e-06 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.291 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171583+0000", 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7985794429999999 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171651+0000", 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8609000000000007e-05 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171850+0000", 2024-09-13T14:51:56.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019853899999999999 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179160+0000", 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007310464 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189042+0000", 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098813229999999991 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189161+0000", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011959399999999999 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205674+0000", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016513116000000001 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205809+0000", 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013482699999999999 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205965+0000", 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015622100000000001 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214091 10.17 10:ef747aa2:::benchmark_data_smithi042_191918_object90498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828211309, 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8340415399999999, 2024-09-13T14:51:56.295 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214091 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.296 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:51:56.297 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.297 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.297 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2399999999999999e-07 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372153+0000", 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372158+0000", 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5480000000000001e-06 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372163+0000", 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9779999999999996e-06 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373060+0000", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00089612399999999999 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373064+0000", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.985e-06 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173248+0000", 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8001843879999999 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173364+0000", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116449 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173684+0000", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031991700000000002 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179424+0000", 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057400389999999997 2024-09-13T14:51:56.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189094+0000", 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096700840000000007 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189287+0000", 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000192797 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206016+0000", 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016728845999999999 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206077+0000", 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1397999999999997e-05 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206192+0000", 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011492 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.303 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214096 10.17 10:e8b3be78:::benchmark_data_smithi042_191918_object90503:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.828193574, 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.834227032, 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214096 2024-09-13T14:51:56.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.305 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372171+0000", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.699e-06 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372178+0000", 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.08e-06 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372183+0000", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.541e-06 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373085+0000", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00090176500000000001 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373091+0000", 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0850000000000002e-06 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176017+0000", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.802925707 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176142+0000", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000125635 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176503+0000", 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00036053799999999998 2024-09-13T14:51:56.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179430+0000", 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029274700000000002 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189346+0000", 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0099155010000000002 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192704+0000", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033585160000000002 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206245+0000", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013540942 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206298+0000", 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2794999999999999e-05 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206396+0000", 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7423999999999994e-05 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214183 10.17 10:e94dc7de:::benchmark_data_smithi042_191918_object90590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.757929792, 2024-09-13T14:51:56.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.764709037, 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214183 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442434+0000", 2024-09-13T14:51:56.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3120000000000001e-06 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442442+0000", 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3610000000000001e-06 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442452+0000", 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.008e-05 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442457+0000", 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9819999999999999e-06 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442498+0000", 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0735000000000002e-05 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442504+0000", 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6039999999999998e-06 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179503+0000", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7369989829999999 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179571+0000", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7988e-05 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179886+0000", 2024-09-13T14:51:56.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031447400000000003 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184110+0000", 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042242970000000001 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189409+0000", 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052988510000000003 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199205+0000", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097961820000000005 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206437+0000", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072323140000000001 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206483+0000", 2024-09-13T14:51:56.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5970000000000002e-05 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207141+0000", 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00065790399999999995 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214185 10.17 10:e975695b:::benchmark_data_smithi042_191918_object90592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.319 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.757020815, 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7640301379999999, 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214185 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443342+0000", 2024-09-13T14:51:56.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3300000000000002e-07 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443345+0000", 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.225e-06 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443353+0000", 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0190000000000006e-06 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443357+0000", 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3100000000000002e-06 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443396+0000", 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8965999999999999e-05 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443401+0000", 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8239999999999999e-06 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.323 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.181721+0000", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7383201939999999 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.181835+0000", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113781 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.182076+0000", 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000240635 2024-09-13T14:51:56.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.186916+0000", 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048399890000000003 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189459+0000", 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025427190000000001 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199247+0000", 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097884879999999997 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207190+0000", 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079428859999999997 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207248+0000", 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.766e-05 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207371+0000", 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012380900000000001 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214190 10.17 10:ecea52bb:::benchmark_data_smithi042_191918_object90597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.74977729299999, 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7569722080000001, 2024-09-13T14:51:56.327 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214190 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450586+0000", 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3009999999999999e-06 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450591+0000", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9710000000000003e-06 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450600+0000", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7900000000000005e-06 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450605+0000", 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-06 2024-09-13T14:51:56.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450658+0000", 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3009e-05 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450668+0000", 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8630000000000003e-06 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184291+0000", 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7336238420000001 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184365+0000", 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3829000000000001e-05 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184662+0000", 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000296345 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189177+0000", 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045157039999999997 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189500+0000", 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00032273399999999998 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199282+0000", 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097823219999999995 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207411+0000", 2024-09-13T14:51:56.334 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081288769999999996 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207459+0000", 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7642999999999998e-05 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207557+0000", 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8128000000000006e-05 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.335 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214243 10.17 10:ee61d6bb:::benchmark_data_smithi042_191918_object90650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.63765938100001, 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.645319298, 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.336 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214243 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:51:56.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562704+0000", 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.637e-06 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562709+0000", 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3460000000000004e-06 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562718+0000", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8799999999999997e-06 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562723+0000", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9450000000000001e-06 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562779+0000", 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5989999999999998e-05 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562789+0000", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0151e-05 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.186965+0000", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.62417641 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.187065+0000", 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100016 2024-09-13T14:51:56.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.187302+0000", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023705800000000001 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189364+0000", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020619779999999999 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192310+0000", 2024-09-13T14:51:56.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029459080000000001 2024-09-13T14:51:56.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199326+0000", 2024-09-13T14:51:56.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0070158210000000002 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207595+0000", 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082688569999999992 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207642+0000", 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6981000000000003e-05 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208022+0000", 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00038032000000000002 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214298 10.17 10:ebc9c796:::benchmark_data_smithi042_191918_object90705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 120.379875924, 2024-09-13T14:51:56.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3877340249999999, 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214298 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820487+0000", 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2899999999999999e-06 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820492+0000", 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3059999999999999e-06 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820501+0000", 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5279999999999992e-06 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820507+0000", 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820561+0000", 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4466000000000003e-05 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820571+0000", 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:51:56.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189546+0000", 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3689755100000001 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189656+0000", 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109638 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:51:56.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.190003+0000", 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034756400000000002 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192406+0000", 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002402141 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192450+0000", 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4297e-05 2024-09-13T14:51:56.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199916+0000", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007466215 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208069+0000", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081527119999999995 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208121+0000", 2024-09-13T14:51:56.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2392000000000003e-05 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208220+0000", 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9061999999999994e-05 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:45221 11.9 11:9e90290f:::benchmark_data_smithi042_223320_object45220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:51:56.352 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.816229+0000", 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 2.384133544, 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0608810280000001, 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 45221 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.816229+0000", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.816229+0000", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.816230+0000", 2024-09-13T14:51:56.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6130000000000001e-06 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.816260+0000", 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9363999999999999e-05 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.861154+0000", 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.044894525999999997 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.861171+0000", 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7172000000000001e-05 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.861904+0000", 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00073297100000000001 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.862103+0000", 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000199147 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.862298+0000", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019455899999999999 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.864429+0000", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021308659999999999 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.865954+0000", 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015254209999999999 2024-09-13T14:51:56.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.875130+0000", 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0091762340000000001 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.875141+0000", 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0482999999999999e-05 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.877019+0000", 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0018782420000001 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.877110+0000", 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.043e-05 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.359 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:45233 11.9 11:977af44c:::benchmark_data_smithi042_223320_object45232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.867143+0000", 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 2.3332191409999998, 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.013972087, 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 45233 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:51:56.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867143+0000", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867143+0000", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867144+0000", 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.378e-06 2024-09-13T14:51:56.361 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867166+0000", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1481999999999999e-05 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867184+0000", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8110000000000001e-05 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867192+0000", 2024-09-13T14:51:56.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.551e-06 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867254+0000", 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1303000000000006e-05 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867385+0000", 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000131509 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T14:51:56.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.867532+0000", 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000146489 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.870506+0000", 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029742879999999998 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.875200+0000", 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046937309999999996 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.875211+0000", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1123e-05 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.877358+0000", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.002146913 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for degraded object", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.877761+0000", 2024-09-13T14:51:56.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00040332399999999999 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.881031+0000", 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032695720000000001 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:54.881115+0000", 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4314000000000006e-05 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:51:56.366 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:51:56.367 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:51:56.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:56.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:56.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:56.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:56.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:56.632 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:56.672 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:56.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 48745 48729 117.119 104 0.00656612 0.00853275 2024-09-13T14:51:56.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:56.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:56.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:51:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:56.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:56.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:51:57.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:57.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:57.191 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.323 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:57.349 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:57.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:57.451 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:57.479 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:57.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:57.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:51:57.581 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:57.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:57.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:57.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:57.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 50354 50338 116.505 100.562 0.00638792 0.00857802 2024-09-13T14:51:57.811 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:57.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:57.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:57.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:57.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:58.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:58.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:58.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:58.138 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[113926]: pgmap v887: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:51:58.138 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[113926]: 1.2 scrub starts 2024-09-13T14:51:58.138 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[113926]: 1.2 scrub ok 2024-09-13T14:51:58.139 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[108570]: pgmap v887: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:51:58.139 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[108570]: 1.2 scrub starts 2024-09-13T14:51:58.139 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:57 smithi042 ceph-mon[108570]: 1.2 scrub ok 2024-09-13T14:51:58.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:51:58.165 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:57 smithi159 ceph-mon[102519]: pgmap v887: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s; 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:51:58.165 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:57 smithi159 ceph-mon[102519]: 1.2 scrub starts 2024-09-13T14:51:58.165 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:57 smithi159 ceph-mon[102519]: 1.2 scrub ok 2024-09-13T14:51:58.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:58.233 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:58.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:58.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:58.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:58.459 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:51:58.483 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:58.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:58.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:51:58.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.659 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:58.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:51:58.716 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:58.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:58.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 51918 51902 115.835 97.75 0.0136923 0.00862725 2024-09-13T14:51:58.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:58.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:58.788 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:51:58.792 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:51:58.793 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:51:58.841 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:51:58.841 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:51:58.873 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:51:58.874 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:51:58.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:58.909 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:51:58.909 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:51:58.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:58.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:58.944 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:51:58.944 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:51:58.977 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:51:58.977 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:51:58.978 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1540281397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:58.978 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1540281397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:58.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1540281397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:51:59.012 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:51:59.013 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:51:59.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:59.051 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:51:59.051 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:51:59.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:59.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:59.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:51:59.193 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:59.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:51:59.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:51:59.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:51:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:51:59.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:51:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:59.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:51:59.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:51:59.746 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:59.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:51:59.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 53456 53440 115.155 96.125 0.0151883 0.00867905 2024-09-13T14:51:59.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:51:59.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:51:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:51:59.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:51:59.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:00.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:51:59 smithi042 ceph-mon[108570]: pgmap v888: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:51:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:51:59] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.51.0" 2024-09-13T14:52:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:51:59 smithi042 ceph-mon[113926]: pgmap v888: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:00.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:00.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:51:59 smithi159 ceph-mon[102519]: pgmap v888: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s; 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:52:00.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:00.284 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:00.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:00.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:00.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:00.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:00.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:00.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:00.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:00.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 54931 54915 114.39 92.1875 0.0186322 0.00873628 2024-09-13T14:52:00.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:00.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:00.922 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:00.923 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:00.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:00.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:00.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:01.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:01.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:01.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:01.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:01.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:01.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.508 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:01.544 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:01.646 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:01.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:52:01.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:01.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 56296 56280 113.451 85.3125 0.00569323 0.00880876 2024-09-13T14:52:01.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:01.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:01.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:01.956 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:01.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:01.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:01.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:02.059 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.089 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:02.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:02.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.245 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:01 smithi159 ceph-mon[102519]: pgmap v889: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.8 MiB/s, 28 objects/s recovering 2024-09-13T14:52:02.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:02.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:01 smithi042 ceph-mon[113926]: pgmap v889: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.8 MiB/s, 28 objects/s recovering 2024-09-13T14:52:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:01 smithi042 ceph-mon[108570]: pgmap v889: 99 pgs: 99 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 1.8 MiB/s, 28 objects/s recovering 2024-09-13T14:52:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:02.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:02.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:02.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.492 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:02.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:02.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:02.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.632 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:02.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:02.734 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:02.760 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:02.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 57797 57781 112.837 93.8125 0.0105052 0.00885675 2024-09-13T14:52:02.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:02.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:02.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:02.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:02.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.026 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:03.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:03.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:03.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:03.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:03.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:03.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:03.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:03.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.574 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:03.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:03.708 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:03.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 15 59316 59301 112.297 95 0.00912752 0.0088996 2024-09-13T14:52:03.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:03.831 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:03.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:03.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:03.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:03.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:04.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:04.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:04.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:03 smithi159 ceph-mon[102519]: pgmap v890: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:52:04.273 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:03 smithi042 ceph-mon[113926]: pgmap v890: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:52:04.273 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:03 smithi042 ceph-mon[108570]: pgmap v890: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s; 1.7 MiB/s, 26 objects/s recovering 2024-09-13T14:52:04.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:04.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:04.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.647 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 60526 60510 111.216 75.5625 0.0146348 0.00898618 2024-09-13T14:52:04.788 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:04.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:04.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:04.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:05.020 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:05.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:05.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:05.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:05.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:05.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:05.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:05.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:05.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:05.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:05.561 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:05.562 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:05.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:05.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:05.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:05.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:05.763 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 61706 61690 110.145 73.75 0.00641826 0.00907378 2024-09-13T14:52:05.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:05.848 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:05.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:05.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:05.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:05.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:05.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:06.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:05 smithi042 ceph-mon[108570]: pgmap v891: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:52:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:05 smithi042 ceph-mon[113926]: pgmap v891: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:52:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.124 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:06.225 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:05 smithi159 ceph-mon[102519]: pgmap v891: 99 pgs: 99 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s; 1.4 MiB/s, 22 objects/s recovering 2024-09-13T14:52:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:06.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:06.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:06.359 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:06.397 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:06.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.544 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:06.646 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:06.688 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:06.763 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 63211 63195 109.698 94.0625 0.00628839 0.00911051 2024-09-13T14:52:06.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:06.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:06.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:06.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:06.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:07.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:07.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:07.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:07.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:07.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.375 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:07.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.477 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:07.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:52:07.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:07.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:07.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:07.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:07.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:07.762 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 64676 64660 109.208 91.5625 0.00841547 0.00915239 2024-09-13T14:52:07.792 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:07.812 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:07.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:07.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:07.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:07.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:08.047 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:08.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:08.155 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:07 smithi159 ceph-mon[102519]: pgmap v892: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:52:08.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:08.234 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:08.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:08.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:07 smithi042 ceph-mon[108570]: pgmap v892: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:52:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:07 smithi042 ceph-mon[113926]: pgmap v892: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:52:08.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.367 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:08.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:08.468 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:08.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:08.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.643 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:08.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.763 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 65981 65965 108.48 81.5625 0.0162499 0.00921218 2024-09-13T14:52:08.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:08.872 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:08.902 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:08.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.001 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:09.001 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:09.004 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:09.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:09.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:09.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.163 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:52:09.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:09.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:09.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:09.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:09.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:09.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:09.625 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:52:09.626 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:52:09.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:09.757 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:09.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:09.764 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 67129 67113 107.538 71.75 0.0224375 0.00929354 2024-09-13T14:52:09.808 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:09.808 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:09.859 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:09.882 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:09.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:09.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:09.916 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:09 smithi042 ceph-mon[108570]: pgmap v893: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:52:09.917 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:09] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-13T14:52:09.993 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.026 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.127 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:09 smithi159 ceph-mon[102519]: pgmap v893: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:52:10.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.286 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:09 smithi042 ceph-mon[113926]: pgmap v893: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s 2024-09-13T14:52:10.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.551 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.574 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":893,"stamp":"2024-09-13T14:52:08.382425+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4141291493,"num_objects":63208,"num_object_clones":0,"num_object_copies":189624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63246,"num_write_kb":4043681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1208,"num_bytes_recovered":77333800,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":64880,"ondisk_log_size":64880,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17505868,"kb_used_data":12078476,"kb_used_omap":28,"kb_used_meta":5427299,"kb_avail":732455348,"statfs":{"total":767960285184,"available":750034276352,"internally_reserved":0,"allocated":12368359424,"data_stored":12357295746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":5557554577},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1232744026,"num_objects":18810,"num_object_clones":0,"num_object_copies":56430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18810,"num_whiteouts":0,"num_read":-36,"num_read_kb":-76,"num_write":18791,"num_write_kb":1203684,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003495"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":343,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866670+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:51:54.866670+0000","last_peered":"2024-09-13T14:51:54.866670+0000","last_clean":"2024-09-13T14:51:54.866670+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:51:54.866670+0000","last_undegraded":"2024-09-13T14:51:54.866670+0000","last_fullsized":"2024-09-13T14:51:54.866670+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":320,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867819+0000","last_change":"2024-09-13T14:43:35.907052+0000","last_active":"2024-09-13T14:51:54.867819+0000","last_peered":"2024-09-13T14:51:54.867819+0000","last_clean":"2024-09-13T14:51:54.867819+0000","last_became_active":"2024-09-13T14:43:35.906588+0000","last_became_peered":"2024-09-13T14:43:35.906588+0000","last_unstale":"2024-09-13T14:51:54.867819+0000","last_undegraded":"2024-09-13T14:51:54.867819+0000","last_fullsized":"2024-09-13T14:51:54.867819+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:56.521107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.878930+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:51:54.878930+0000","last_peered":"2024-09-13T14:51:54.878930+0000","last_clean":"2024-09-13T14:51:54.878930+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:51:54.878930+0000","last_undegraded":"2024-09-13T14:51:54.878930+0000","last_fullsized":"2024-09-13T14:51:54.878930+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.870943+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:51:54.870943+0000","last_peered":"2024-09-13T14:51:54.870943+0000","last_clean":"2024-09-13T14:51:54.870943+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:51:54.870943+0000","last_undegraded":"2024-09-13T14:51:54.870943+0000","last_fullsized":"2024-09-13T14:51:54.870943+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871061+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:51:54.871061+0000","last_peered":"2024-09-13T14:51:54.871061+0000","last_clean":"2024-09-13T14:51:54.871061+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:51:54.871061+0000","last_undegraded":"2024-09-13T14:51:54.871061+0000","last_fullsized":"2024-09-13T14:51:54.871061+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":119,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877371+0000","last_change":"2024-09-13T14:43:35.907589+0000","last_active":"2024-09-13T14:51:53.877371+0000","last_peered":"2024-09-13T14:51:53.877371+0000","last_clean":"2024-09-13T14:51:53.877371+0000","last_became_active":"2024-09-13T14:43:35.907254+0000","last_became_peered":"2024-09-13T14:43:35.907254+0000","last_unstale":"2024-09-13T14:51:53.877371+0000","last_undegraded":"2024-09-13T14:51:53.877371+0000","last_fullsized":"2024-09-13T14:51:53.877371+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:13.369649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":369,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866300+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:51:54.866300+0000","last_peered":"2024-09-13T14:51:54.866300+0000","last_clean":"2024-09-13T14:51:54.866300+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:51:54.866300+0000","last_undegraded":"2024-09-13T14:51:54.866300+0000","last_fullsized":"2024-09-13T14:51:54.866300+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":295,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.872110+0000","last_change":"2024-09-13T14:51:37.799522+0000","last_active":"2024-09-13T14:51:53.872110+0000","last_peered":"2024-09-13T14:51:53.872110+0000","last_clean":"2024-09-13T14:51:53.872110+0000","last_became_active":"2024-09-13T14:51:37.789544+0000","last_became_peered":"2024-09-13T14:51:37.789544+0000","last_unstale":"2024-09-13T14:51:53.872110+0000","last_undegraded":"2024-09-13T14:51:53.872110+0000","last_fullsized":"2024-09-13T14:51:53.872110+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:27.557594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"261'3887","reported_seq":3912,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.403040+0000","last_change":"2024-09-13T14:51:16.446237+0000","last_active":"2024-09-13T14:52:06.403040+0000","last_peered":"2024-09-13T14:52:06.403040+0000","last_clean":"2024-09-13T14:52:06.403040+0000","last_became_active":"2024-09-13T14:51:16.445868+0000","last_became_peered":"2024-09-13T14:51:16.445868+0000","last_unstale":"2024-09-13T14:52:06.403040+0000","last_undegraded":"2024-09-13T14:52:06.403040+0000","last_fullsized":"2024-09-13T14:52:06.403040+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3887,"log_dups_size":0,"ondisk_log_size":3887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:14.921393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254738432,"num_objects":3887,"num_object_clones":0,"num_object_copies":11661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3887,"num_write_kb":248768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":170,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877999+0000","last_change":"2024-09-13T14:51:37.793374+0000","last_active":"2024-09-13T14:51:53.877999+0000","last_peered":"2024-09-13T14:51:53.877999+0000","last_clean":"2024-09-13T14:51:53.877999+0000","last_became_active":"2024-09-13T14:51:37.788764+0000","last_became_peered":"2024-09-13T14:51:37.788764+0000","last_unstale":"2024-09-13T14:51:53.877999+0000","last_undegraded":"2024-09-13T14:51:53.877999+0000","last_fullsized":"2024-09-13T14:51:53.877999+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:44.417775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":214,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876866+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:51:53.876866+0000","last_peered":"2024-09-13T14:51:53.876866+0000","last_clean":"2024-09-13T14:51:53.876866+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:51:53.876866+0000","last_undegraded":"2024-09-13T14:51:53.876866+0000","last_fullsized":"2024-09-13T14:51:53.876866+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"261'4154","reported_seq":4191,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:07.825741+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:52:07.825741+0000","last_peered":"2024-09-13T14:52:07.825741+0000","last_clean":"2024-09-13T14:52:07.825741+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:52:07.825741+0000","last_undegraded":"2024-09-13T14:52:07.825741+0000","last_fullsized":"2024-09-13T14:52:07.825741+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:44.438538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272236544,"num_objects":4154,"num_object_clones":0,"num_object_copies":12462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4154,"num_write_kb":265856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":131,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877881+0000","last_change":"2024-09-13T14:43:35.905294+0000","last_active":"2024-09-13T14:51:53.877881+0000","last_peered":"2024-09-13T14:51:53.877881+0000","last_clean":"2024-09-13T14:51:53.877881+0000","last_became_active":"2024-09-13T14:43:35.904851+0000","last_became_peered":"2024-09-13T14:43:35.904851+0000","last_unstale":"2024-09-13T14:51:53.877881+0000","last_undegraded":"2024-09-13T14:51:53.877881+0000","last_fullsized":"2024-09-13T14:51:53.877881+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:21.591225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877331+0000","last_change":"2024-09-13T14:40:37.187755+0000","last_active":"2024-09-13T14:51:53.877331+0000","last_peered":"2024-09-13T14:51:53.877331+0000","last_clean":"2024-09-13T14:51:53.877331+0000","last_became_active":"2024-09-13T14:40:37.187373+0000","last_became_peered":"2024-09-13T14:40:37.187373+0000","last_unstale":"2024-09-13T14:51:53.877331+0000","last_undegraded":"2024-09-13T14:51:53.877331+0000","last_fullsized":"2024-09-13T14:51:53.877331+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:10:43.966583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"261'3865","reported_seq":3891,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.401384+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:52:06.401384+0000","last_peered":"2024-09-13T14:52:06.401384+0000","last_clean":"2024-09-13T14:52:06.401384+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:52:06.401384+0000","last_undegraded":"2024-09-13T14:52:06.401384+0000","last_fullsized":"2024-09-13T14:52:06.401384+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3865,"log_dups_size":0,"ondisk_log_size":3865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253296640,"num_objects":3865,"num_object_clones":0,"num_object_copies":11595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3865,"num_write_kb":247360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":131,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876527+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:51:53.876527+0000","last_peered":"2024-09-13T14:51:53.876527+0000","last_clean":"2024-09-13T14:51:53.876527+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:51:53.876527+0000","last_undegraded":"2024-09-13T14:51:53.876527+0000","last_fullsized":"2024-09-13T14:51:53.876527+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.871268+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:51:53.871268+0000","last_peered":"2024-09-13T14:51:53.871268+0000","last_clean":"2024-09-13T14:51:53.871268+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:51:53.871268+0000","last_undegraded":"2024-09-13T14:51:53.871268+0000","last_fullsized":"2024-09-13T14:51:53.871268+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"261'3964","reported_seq":3990,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.400832+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:52:06.400832+0000","last_peered":"2024-09-13T14:52:06.400832+0000","last_clean":"2024-09-13T14:52:06.400832+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:52:06.400832+0000","last_undegraded":"2024-09-13T14:52:06.400832+0000","last_fullsized":"2024-09-13T14:52:06.400832+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3964,"log_dups_size":0,"ondisk_log_size":3964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259784704,"num_objects":3964,"num_object_clones":0,"num_object_copies":11892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3964,"num_write_kb":253696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876631+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:51:53.876631+0000","last_peered":"2024-09-13T14:51:53.876631+0000","last_clean":"2024-09-13T14:51:53.876631+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:51:53.876631+0000","last_undegraded":"2024-09-13T14:51:53.876631+0000","last_fullsized":"2024-09-13T14:51:53.876631+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871268+0000","last_change":"2024-09-13T14:51:20.405103+0000","last_active":"2024-09-13T14:51:54.871268+0000","last_peered":"2024-09-13T14:51:54.871268+0000","last_clean":"2024-09-13T14:51:54.871268+0000","last_became_active":"2024-09-13T14:51:20.404344+0000","last_became_peered":"2024-09-13T14:51:20.404344+0000","last_unstale":"2024-09-13T14:51:54.871268+0000","last_undegraded":"2024-09-13T14:51:54.871268+0000","last_fullsized":"2024-09-13T14:51:54.871268+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:05.332557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"261'3967","reported_seq":3993,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.421250+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:52:06.421250+0000","last_peered":"2024-09-13T14:52:06.421250+0000","last_clean":"2024-09-13T14:52:06.421250+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:52:06.421250+0000","last_undegraded":"2024-09-13T14:52:06.421250+0000","last_fullsized":"2024-09-13T14:52:06.421250+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3967,"log_dups_size":0,"ondisk_log_size":3967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259981312,"num_objects":3967,"num_object_clones":0,"num_object_copies":11901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3967,"num_write_kb":253888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":693,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.864845+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:51:54.864845+0000","last_peered":"2024-09-13T14:51:54.864845+0000","last_clean":"2024-09-13T14:51:54.864845+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:51:54.864845+0000","last_undegraded":"2024-09-13T14:51:54.864845+0000","last_fullsized":"2024-09-13T14:51:54.864845+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":365,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865672+0000","last_change":"2024-09-13T14:51:37.799188+0000","last_active":"2024-09-13T14:51:54.865672+0000","last_peered":"2024-09-13T14:51:54.865672+0000","last_clean":"2024-09-13T14:51:54.865672+0000","last_became_active":"2024-09-13T14:51:37.792663+0000","last_became_peered":"2024-09-13T14:51:37.792663+0000","last_unstale":"2024-09-13T14:51:54.865672+0000","last_undegraded":"2024-09-13T14:51:54.865672+0000","last_fullsized":"2024-09-13T14:51:54.865672+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:04:04.567415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877494+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:51:53.877494+0000","last_peered":"2024-09-13T14:51:53.877494+0000","last_clean":"2024-09-13T14:51:53.877494+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:51:53.877494+0000","last_undegraded":"2024-09-13T14:51:53.877494+0000","last_fullsized":"2024-09-13T14:51:53.877494+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"261'3856","reported_seq":3908,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.094494+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:52:06.094494+0000","last_peered":"2024-09-13T14:52:06.094494+0000","last_clean":"2024-09-13T14:52:06.094494+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:52:06.094494+0000","last_undegraded":"2024-09-13T14:52:06.094494+0000","last_fullsized":"2024-09-13T14:52:06.094494+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3856,"log_dups_size":0,"ondisk_log_size":3856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:01.898177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252706816,"num_objects":3856,"num_object_clones":0,"num_object_copies":11568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3856,"num_write_kb":246784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":499,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876655+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:51:53.876655+0000","last_peered":"2024-09-13T14:51:53.876655+0000","last_clean":"2024-09-13T14:51:53.876655+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:51:53.876655+0000","last_undegraded":"2024-09-13T14:51:53.876655+0000","last_fullsized":"2024-09-13T14:51:53.876655+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":251,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.871469+0000","last_change":"2024-09-13T14:40:38.174853+0000","last_active":"2024-09-13T14:51:53.871469+0000","last_peered":"2024-09-13T14:51:53.871469+0000","last_clean":"2024-09-13T14:51:53.871469+0000","last_became_active":"2024-09-13T14:40:38.174484+0000","last_became_peered":"2024-09-13T14:40:38.174484+0000","last_unstale":"2024-09-13T14:51:53.871469+0000","last_undegraded":"2024-09-13T14:51:53.871469+0000","last_fullsized":"2024-09-13T14:51:53.871469+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:37:30.574928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877778+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:51:53.877778+0000","last_peered":"2024-09-13T14:51:53.877778+0000","last_clean":"2024-09-13T14:51:53.877778+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:51:53.877778+0000","last_undegraded":"2024-09-13T14:51:53.877778+0000","last_fullsized":"2024-09-13T14:51:53.877778+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"261'3794","reported_seq":3829,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.414451+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:52:06.414451+0000","last_peered":"2024-09-13T14:52:06.414451+0000","last_clean":"2024-09-13T14:52:06.414451+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:52:06.414451+0000","last_undegraded":"2024-09-13T14:52:06.414451+0000","last_fullsized":"2024-09-13T14:52:06.414451+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3794,"log_dups_size":0,"ondisk_log_size":3794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248643584,"num_objects":3794,"num_object_clones":0,"num_object_copies":11382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3794,"num_write_kb":242816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877419+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:51:53.877419+0000","last_peered":"2024-09-13T14:51:53.877419+0000","last_clean":"2024-09-13T14:51:53.877419+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:51:53.877419+0000","last_undegraded":"2024-09-13T14:51:53.877419+0000","last_fullsized":"2024-09-13T14:51:53.877419+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":238,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877891+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:51:53.877891+0000","last_peered":"2024-09-13T14:51:53.877891+0000","last_clean":"2024-09-13T14:51:53.877891+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:51:53.877891+0000","last_undegraded":"2024-09-13T14:51:53.877891+0000","last_fullsized":"2024-09-13T14:51:53.877891+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"261'4049","reported_seq":4094,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:08.306939+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:52:08.306939+0000","last_peered":"2024-09-13T14:52:08.306939+0000","last_clean":"2024-09-13T14:52:08.306939+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:52:08.306939+0000","last_undegraded":"2024-09-13T14:52:08.306939+0000","last_fullsized":"2024-09-13T14:52:08.306939+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":4049,"log_dups_size":0,"ondisk_log_size":4049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:42:21.557196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265355264,"num_objects":4049,"num_object_clones":0,"num_object_copies":12147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4049,"num_write_kb":259136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":499,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876112+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:51:53.876112+0000","last_peered":"2024-09-13T14:51:53.876112+0000","last_clean":"2024-09-13T14:51:53.876112+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:51:53.876112+0000","last_undegraded":"2024-09-13T14:51:53.876112+0000","last_fullsized":"2024-09-13T14:51:53.876112+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":346,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866097+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:51:54.866097+0000","last_peered":"2024-09-13T14:51:54.866097+0000","last_clean":"2024-09-13T14:51:54.866097+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:51:54.866097+0000","last_undegraded":"2024-09-13T14:51:54.866097+0000","last_fullsized":"2024-09-13T14:51:54.866097+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":345,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.868210+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:51:54.868210+0000","last_peered":"2024-09-13T14:51:54.868210+0000","last_clean":"2024-09-13T14:51:54.868210+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:51:54.868210+0000","last_undegraded":"2024-09-13T14:51:54.868210+0000","last_fullsized":"2024-09-13T14:51:54.868210+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"261'3894","reported_seq":3939,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:05.980110+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:52:05.980110+0000","last_peered":"2024-09-13T14:52:05.980110+0000","last_clean":"2024-09-13T14:52:05.980110+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:52:05.980110+0000","last_undegraded":"2024-09-13T14:52:05.980110+0000","last_fullsized":"2024-09-13T14:52:05.980110+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3894,"log_dups_size":0,"ondisk_log_size":3894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:40.814877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3894,"num_write_kb":249216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":486,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.872110+0000","last_change":"2024-09-13T14:51:40.799562+0000","last_active":"2024-09-13T14:51:53.872110+0000","last_peered":"2024-09-13T14:51:53.872110+0000","last_clean":"2024-09-13T14:51:53.872110+0000","last_became_active":"2024-09-13T14:51:40.799144+0000","last_became_peered":"2024-09-13T14:51:40.799144+0000","last_unstale":"2024-09-13T14:51:53.872110+0000","last_undegraded":"2024-09-13T14:51:53.872110+0000","last_fullsized":"2024-09-13T14:51:53.872110+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:01.166260+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":244,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876698+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:51:53.876698+0000","last_peered":"2024-09-13T14:51:53.876698+0000","last_clean":"2024-09-13T14:51:53.876698+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:51:53.876698+0000","last_undegraded":"2024-09-13T14:51:53.876698+0000","last_fullsized":"2024-09-13T14:51:53.876698+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":331,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866427+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:51:54.866427+0000","last_peered":"2024-09-13T14:51:54.866427+0000","last_clean":"2024-09-13T14:51:54.866427+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:51:54.866427+0000","last_undegraded":"2024-09-13T14:51:54.866427+0000","last_fullsized":"2024-09-13T14:51:54.866427+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":215,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.871619+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:51:53.871619+0000","last_peered":"2024-09-13T14:51:53.871619+0000","last_clean":"2024-09-13T14:51:53.871619+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:51:53.871619+0000","last_undegraded":"2024-09-13T14:51:53.871619+0000","last_fullsized":"2024-09-13T14:51:53.871619+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"261'3877","reported_seq":6301,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.013999+0000","last_change":"2024-09-13T14:51:54.914091+0000","last_active":"2024-09-13T14:52:06.013999+0000","last_peered":"2024-09-13T14:52:06.013999+0000","last_clean":"2024-09-13T14:52:06.013999+0000","last_became_active":"2024-09-13T14:51:54.875147+0000","last_became_peered":"2024-09-13T14:51:54.875147+0000","last_unstale":"2024-09-13T14:52:06.013999+0000","last_undegraded":"2024-09-13T14:52:06.013999+0000","last_fullsized":"2024-09-13T14:52:06.013999+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3877,"log_dups_size":0,"ondisk_log_size":3877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:58:02.053672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254083072,"num_objects":3877,"num_object_clones":0,"num_object_copies":11631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3877,"num_write_kb":248128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":605,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867689+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:51:54.867689+0000","last_peered":"2024-09-13T14:51:54.867689+0000","last_clean":"2024-09-13T14:51:54.867689+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:51:54.867689+0000","last_undegraded":"2024-09-13T14:51:54.867689+0000","last_fullsized":"2024-09-13T14:51:54.867689+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":332,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865481+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:51:54.865481+0000","last_peered":"2024-09-13T14:51:54.865481+0000","last_clean":"2024-09-13T14:51:54.865481+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:51:54.865481+0000","last_undegraded":"2024-09-13T14:51:54.865481+0000","last_fullsized":"2024-09-13T14:51:54.865481+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.872178+0000","last_change":"2024-09-13T14:51:20.405045+0000","last_active":"2024-09-13T14:51:54.872178+0000","last_peered":"2024-09-13T14:51:54.872178+0000","last_clean":"2024-09-13T14:51:54.872178+0000","last_became_active":"2024-09-13T14:51:20.404726+0000","last_became_peered":"2024-09-13T14:51:20.404726+0000","last_unstale":"2024-09-13T14:51:54.872178+0000","last_undegraded":"2024-09-13T14:51:54.872178+0000","last_fullsized":"2024-09-13T14:51:54.872178+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:04.185004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":278,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867754+0000","last_change":"2024-09-13T14:51:37.792733+0000","last_active":"2024-09-13T14:51:54.867754+0000","last_peered":"2024-09-13T14:51:54.867754+0000","last_clean":"2024-09-13T14:51:54.867754+0000","last_became_active":"2024-09-13T14:51:37.791658+0000","last_became_peered":"2024-09-13T14:51:37.791658+0000","last_unstale":"2024-09-13T14:51:54.867754+0000","last_undegraded":"2024-09-13T14:51:54.867754+0000","last_fullsized":"2024-09-13T14:51:54.867754+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:10.654606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"261'3835","reported_seq":3880,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.023445+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:52:06.023445+0000","last_peered":"2024-09-13T14:52:06.023445+0000","last_clean":"2024-09-13T14:52:06.023445+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:52:06.023445+0000","last_undegraded":"2024-09-13T14:52:06.023445+0000","last_fullsized":"2024-09-13T14:52:06.023445+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3835,"log_dups_size":0,"ondisk_log_size":3835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:47.882614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251330560,"num_objects":3835,"num_object_clones":0,"num_object_copies":11505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3835,"num_write_kb":245440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":523,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:57.042073+0000","last_change":"2024-09-13T14:51:57.042073+0000","last_active":"2024-09-13T14:51:57.042073+0000","last_peered":"2024-09-13T14:51:57.042073+0000","last_clean":"2024-09-13T14:51:57.042073+0000","last_became_active":"2024-09-13T14:51:23.788033+0000","last_became_peered":"2024-09-13T14:51:23.788033+0000","last_unstale":"2024-09-13T14:51:57.042073+0000","last_undegraded":"2024-09-13T14:51:57.042073+0000","last_fullsized":"2024-09-13T14:51:57.042073+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:44:43.977371+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":383,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865326+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:51:54.865326+0000","last_peered":"2024-09-13T14:51:54.865326+0000","last_clean":"2024-09-13T14:51:54.865326+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:51:54.865326+0000","last_undegraded":"2024-09-13T14:51:54.865326+0000","last_fullsized":"2024-09-13T14:51:54.865326+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":344,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865885+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:51:54.865885+0000","last_peered":"2024-09-13T14:51:54.865885+0000","last_clean":"2024-09-13T14:51:54.865885+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:51:54.865885+0000","last_undegraded":"2024-09-13T14:51:54.865885+0000","last_fullsized":"2024-09-13T14:51:54.865885+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":394,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.868124+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:51:54.868124+0000","last_peered":"2024-09-13T14:51:54.868124+0000","last_clean":"2024-09-13T14:51:54.868124+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:51:54.868124+0000","last_undegraded":"2024-09-13T14:51:54.868124+0000","last_fullsized":"2024-09-13T14:51:54.868124+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"261'3861","reported_seq":3913,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.012029+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:52:06.012029+0000","last_peered":"2024-09-13T14:52:06.012029+0000","last_clean":"2024-09-13T14:52:06.012029+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:52:06.012029+0000","last_undegraded":"2024-09-13T14:52:06.012029+0000","last_fullsized":"2024-09-13T14:52:06.012029+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3861,"log_dups_size":0,"ondisk_log_size":3861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:56.627626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253034496,"num_objects":3861,"num_object_clones":0,"num_object_copies":11583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3861,"num_write_kb":247104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":533,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876176+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:51:53.876176+0000","last_peered":"2024-09-13T14:51:53.876176+0000","last_clean":"2024-09-13T14:51:53.876176+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:51:53.876176+0000","last_undegraded":"2024-09-13T14:51:53.876176+0000","last_fullsized":"2024-09-13T14:51:53.876176+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876387+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:51:53.876387+0000","last_peered":"2024-09-13T14:51:53.876387+0000","last_clean":"2024-09-13T14:51:53.876387+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:51:53.876387+0000","last_undegraded":"2024-09-13T14:51:53.876387+0000","last_fullsized":"2024-09-13T14:51:53.876387+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":282,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865507+0000","last_change":"2024-09-13T14:51:20.411788+0000","last_active":"2024-09-13T14:51:54.865507+0000","last_peered":"2024-09-13T14:51:54.865507+0000","last_clean":"2024-09-13T14:51:54.865507+0000","last_became_active":"2024-09-13T14:51:20.411434+0000","last_became_peered":"2024-09-13T14:51:20.411434+0000","last_unstale":"2024-09-13T14:51:54.865507+0000","last_undegraded":"2024-09-13T14:51:54.865507+0000","last_fullsized":"2024-09-13T14:51:54.865507+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:14.125924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":356,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867596+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:51:54.867596+0000","last_peered":"2024-09-13T14:51:54.867596+0000","last_clean":"2024-09-13T14:51:54.867596+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:51:54.867596+0000","last_undegraded":"2024-09-13T14:51:54.867596+0000","last_fullsized":"2024-09-13T14:51:54.867596+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"261'3943","reported_seq":3988,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.114032+0000","last_change":"2024-09-13T14:51:16.448773+0000","last_active":"2024-09-13T14:52:06.114032+0000","last_peered":"2024-09-13T14:52:06.114032+0000","last_clean":"2024-09-13T14:52:06.114032+0000","last_became_active":"2024-09-13T14:51:16.448426+0000","last_became_peered":"2024-09-13T14:51:16.448426+0000","last_unstale":"2024-09-13T14:52:06.114032+0000","last_undegraded":"2024-09-13T14:52:06.114032+0000","last_fullsized":"2024-09-13T14:52:06.114032+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3943,"log_dups_size":0,"ondisk_log_size":3943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:56.203908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258408448,"num_objects":3943,"num_object_clones":0,"num_object_copies":11829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3943,"num_write_kb":252352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":507,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.871736+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:51:53.871736+0000","last_peered":"2024-09-13T14:51:53.871736+0000","last_clean":"2024-09-13T14:51:53.871736+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:51:53.871736+0000","last_undegraded":"2024-09-13T14:51:53.871736+0000","last_fullsized":"2024-09-13T14:51:53.871736+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":245,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876605+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:51:53.876605+0000","last_peered":"2024-09-13T14:51:53.876605+0000","last_clean":"2024-09-13T14:51:53.876605+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:51:53.876605+0000","last_undegraded":"2024-09-13T14:51:53.876605+0000","last_fullsized":"2024-09-13T14:51:53.876605+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":408,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867792+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:51:54.867792+0000","last_peered":"2024-09-13T14:51:54.867792+0000","last_clean":"2024-09-13T14:51:54.867792+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:51:54.867792+0000","last_undegraded":"2024-09-13T14:51:54.867792+0000","last_fullsized":"2024-09-13T14:51:54.867792+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":265,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865898+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:51:54.865898+0000","last_peered":"2024-09-13T14:51:54.865898+0000","last_clean":"2024-09-13T14:51:54.865898+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:51:54.865898+0000","last_undegraded":"2024-09-13T14:51:54.865898+0000","last_fullsized":"2024-09-13T14:51:54.865898+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"261'4122","reported_seq":4167,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:08.315769+0000","last_change":"2024-09-13T14:51:16.448079+0000","last_active":"2024-09-13T14:52:08.315769+0000","last_peered":"2024-09-13T14:52:08.315769+0000","last_clean":"2024-09-13T14:52:08.315769+0000","last_became_active":"2024-09-13T14:51:16.447056+0000","last_became_peered":"2024-09-13T14:51:16.447056+0000","last_unstale":"2024-09-13T14:52:08.315769+0000","last_undegraded":"2024-09-13T14:52:08.315769+0000","last_fullsized":"2024-09-13T14:52:08.315769+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":4122,"log_dups_size":0,"ondisk_log_size":4122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:30.905220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270139392,"num_objects":4122,"num_object_clones":0,"num_object_copies":12366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4122,"num_write_kb":263808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":541,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.871346+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:51:53.871346+0000","last_peered":"2024-09-13T14:51:53.871346+0000","last_clean":"2024-09-13T14:51:53.871346+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:51:53.871346+0000","last_undegraded":"2024-09-13T14:51:53.871346+0000","last_fullsized":"2024-09-13T14:51:53.871346+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":343,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867691+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:51:54.867691+0000","last_peered":"2024-09-13T14:51:54.867691+0000","last_clean":"2024-09-13T14:51:54.867691+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:51:54.867691+0000","last_undegraded":"2024-09-13T14:51:54.867691+0000","last_fullsized":"2024-09-13T14:51:54.867691+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.870934+0000","last_change":"2024-09-13T14:51:20.403571+0000","last_active":"2024-09-13T14:51:54.870934+0000","last_peered":"2024-09-13T14:51:54.870934+0000","last_clean":"2024-09-13T14:51:54.870934+0000","last_became_active":"2024-09-13T14:51:20.403244+0000","last_became_peered":"2024-09-13T14:51:20.403244+0000","last_unstale":"2024-09-13T14:51:54.870934+0000","last_undegraded":"2024-09-13T14:51:54.870934+0000","last_fullsized":"2024-09-13T14:51:54.870934+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:08.930829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871177+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:51:54.871177+0000","last_peered":"2024-09-13T14:51:54.871177+0000","last_clean":"2024-09-13T14:51:54.871177+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:51:54.871177+0000","last_undegraded":"2024-09-13T14:51:54.871177+0000","last_fullsized":"2024-09-13T14:51:54.871177+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"261'4129","reported_seq":4174,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:08.338476+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:52:08.338476+0000","last_peered":"2024-09-13T14:52:08.338476+0000","last_clean":"2024-09-13T14:52:08.338476+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:52:08.338476+0000","last_undegraded":"2024-09-13T14:52:08.338476+0000","last_fullsized":"2024-09-13T14:52:08.338476+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:31:39.307786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4129,"num_write_kb":264256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":499,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877860+0000","last_change":"2024-09-13T14:43:00.868011+0000","last_active":"2024-09-13T14:51:53.877860+0000","last_peered":"2024-09-13T14:51:53.877860+0000","last_clean":"2024-09-13T14:51:53.877860+0000","last_became_active":"2024-09-13T14:42:56.283775+0000","last_became_peered":"2024-09-13T14:42:56.283775+0000","last_unstale":"2024-09-13T14:51:53.877860+0000","last_undegraded":"2024-09-13T14:51:53.877860+0000","last_fullsized":"2024-09-13T14:51:53.877860+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:00.867870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:00.867870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:25.806175+0000","scrub_duration":0.0043830680000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":354,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866503+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:51:54.866503+0000","last_peered":"2024-09-13T14:51:54.866503+0000","last_clean":"2024-09-13T14:51:54.866503+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:51:54.866503+0000","last_undegraded":"2024-09-13T14:51:54.866503+0000","last_fullsized":"2024-09-13T14:51:54.866503+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":344,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867998+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:51:54.867998+0000","last_peered":"2024-09-13T14:51:54.867998+0000","last_clean":"2024-09-13T14:51:54.867998+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:51:54.867998+0000","last_undegraded":"2024-09-13T14:51:54.867998+0000","last_fullsized":"2024-09-13T14:51:54.867998+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":350,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865129+0000","last_change":"2024-09-13T14:51:37.793493+0000","last_active":"2024-09-13T14:51:54.865129+0000","last_peered":"2024-09-13T14:51:54.865129+0000","last_clean":"2024-09-13T14:51:54.865129+0000","last_became_active":"2024-09-13T14:51:37.792802+0000","last_became_peered":"2024-09-13T14:51:37.792802+0000","last_unstale":"2024-09-13T14:51:54.865129+0000","last_undegraded":"2024-09-13T14:51:54.865129+0000","last_fullsized":"2024-09-13T14:51:54.865129+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:51.004042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"261'3985","reported_seq":4011,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:52:06.714700+0000","last_change":"2024-09-13T14:51:17.034483+0000","last_active":"2024-09-13T14:52:06.714700+0000","last_peered":"2024-09-13T14:52:06.714700+0000","last_clean":"2024-09-13T14:52:06.714700+0000","last_became_active":"2024-09-13T14:51:17.034153+0000","last_became_peered":"2024-09-13T14:51:17.034153+0000","last_unstale":"2024-09-13T14:52:06.714700+0000","last_undegraded":"2024-09-13T14:52:06.714700+0000","last_fullsized":"2024-09-13T14:52:06.714700+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":3985,"log_dups_size":0,"ondisk_log_size":3985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:25.506330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261160960,"num_objects":3985,"num_object_clones":0,"num_object_copies":11955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3985,"num_write_kb":255040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":604,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865419+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:51:54.865419+0000","last_peered":"2024-09-13T14:51:54.865419+0000","last_clean":"2024-09-13T14:51:54.865419+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:51:54.865419+0000","last_undegraded":"2024-09-13T14:51:54.865419+0000","last_fullsized":"2024-09-13T14:51:54.865419+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":205,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867882+0000","last_change":"2024-09-13T14:51:37.797640+0000","last_active":"2024-09-13T14:51:54.867882+0000","last_peered":"2024-09-13T14:51:54.867882+0000","last_clean":"2024-09-13T14:51:54.867882+0000","last_became_active":"2024-09-13T14:51:37.791624+0000","last_became_peered":"2024-09-13T14:51:37.791624+0000","last_unstale":"2024-09-13T14:51:54.867882+0000","last_undegraded":"2024-09-13T14:51:54.867882+0000","last_fullsized":"2024-09-13T14:51:54.867882+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:34.248753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":244,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871622+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:51:54.871622+0000","last_peered":"2024-09-13T14:51:54.871622+0000","last_clean":"2024-09-13T14:51:54.871622+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:51:54.871622+0000","last_undegraded":"2024-09-13T14:51:54.871622+0000","last_fullsized":"2024-09-13T14:51:54.871622+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":216,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.883260+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:51:54.883260+0000","last_peered":"2024-09-13T14:51:54.883260+0000","last_clean":"2024-09-13T14:51:54.883260+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:51:54.883260+0000","last_undegraded":"2024-09-13T14:51:54.883260+0000","last_fullsized":"2024-09-13T14:51:54.883260+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":229,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877507+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:51:53.877507+0000","last_peered":"2024-09-13T14:51:53.877507+0000","last_clean":"2024-09-13T14:51:53.877507+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:51:53.877507+0000","last_undegraded":"2024-09-13T14:51:53.877507+0000","last_fullsized":"2024-09-13T14:51:53.877507+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":238,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877673+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:51:53.877673+0000","last_peered":"2024-09-13T14:51:53.877673+0000","last_clean":"2024-09-13T14:51:53.877673+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:51:53.877673+0000","last_undegraded":"2024-09-13T14:51:53.877673+0000","last_fullsized":"2024-09-13T14:51:53.877673+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":264,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866055+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:51:54.866055+0000","last_peered":"2024-09-13T14:51:54.866055+0000","last_clean":"2024-09-13T14:51:54.866055+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:51:54.866055+0000","last_undegraded":"2024-09-13T14:51:54.866055+0000","last_fullsized":"2024-09-13T14:51:54.866055+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":289,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866903+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:51:54.866903+0000","last_peered":"2024-09-13T14:51:54.866903+0000","last_clean":"2024-09-13T14:51:54.866903+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:51:54.866903+0000","last_undegraded":"2024-09-13T14:51:54.866903+0000","last_fullsized":"2024-09-13T14:51:54.866903+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":228,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876498+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:51:53.876498+0000","last_peered":"2024-09-13T14:51:53.876498+0000","last_clean":"2024-09-13T14:51:53.876498+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:51:53.876498+0000","last_undegraded":"2024-09-13T14:51:53.876498+0000","last_fullsized":"2024-09-13T14:51:53.876498+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":330,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865030+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:51:54.865030+0000","last_peered":"2024-09-13T14:51:54.865030+0000","last_clean":"2024-09-13T14:51:54.865030+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:51:54.865030+0000","last_undegraded":"2024-09-13T14:51:54.865030+0000","last_fullsized":"2024-09-13T14:51:54.865030+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871226+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:51:54.871226+0000","last_peered":"2024-09-13T14:51:54.871226+0000","last_clean":"2024-09-13T14:51:54.871226+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:51:54.871226+0000","last_undegraded":"2024-09-13T14:51:54.871226+0000","last_fullsized":"2024-09-13T14:51:54.871226+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":343,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865077+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:51:54.865077+0000","last_peered":"2024-09-13T14:51:54.865077+0000","last_clean":"2024-09-13T14:51:54.865077+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:51:54.865077+0000","last_undegraded":"2024-09-13T14:51:54.865077+0000","last_fullsized":"2024-09-13T14:51:54.865077+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":231,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871604+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:51:54.871604+0000","last_peered":"2024-09-13T14:51:54.871604+0000","last_clean":"2024-09-13T14:51:54.871604+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:51:54.871604+0000","last_undegraded":"2024-09-13T14:51:54.871604+0000","last_fullsized":"2024-09-13T14:51:54.871604+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":380,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865222+0000","last_change":"2024-09-13T14:51:37.794313+0000","last_active":"2024-09-13T14:51:54.865222+0000","last_peered":"2024-09-13T14:51:54.865222+0000","last_clean":"2024-09-13T14:51:54.865222+0000","last_became_active":"2024-09-13T14:51:37.792700+0000","last_became_peered":"2024-09-13T14:51:37.792700+0000","last_unstale":"2024-09-13T14:51:54.865222+0000","last_undegraded":"2024-09-13T14:51:54.865222+0000","last_fullsized":"2024-09-13T14:51:54.865222+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:17.804238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":36,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.871061+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:51:54.871061+0000","last_peered":"2024-09-13T14:51:54.871061+0000","last_clean":"2024-09-13T14:51:54.871061+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:51:54.871061+0000","last_undegraded":"2024-09-13T14:51:54.871061+0000","last_fullsized":"2024-09-13T14:51:54.871061+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.867009+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:51:54.867009+0000","last_peered":"2024-09-13T14:51:54.867009+0000","last_clean":"2024-09-13T14:51:54.867009+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:51:54.867009+0000","last_undegraded":"2024-09-13T14:51:54.867009+0000","last_fullsized":"2024-09-13T14:51:54.867009+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":343,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865692+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:51:54.865692+0000","last_peered":"2024-09-13T14:51:54.865692+0000","last_clean":"2024-09-13T14:51:54.865692+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:51:54.865692+0000","last_undegraded":"2024-09-13T14:51:54.865692+0000","last_fullsized":"2024-09-13T14:51:54.865692+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":267,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865358+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:51:54.865358+0000","last_peered":"2024-09-13T14:51:54.865358+0000","last_clean":"2024-09-13T14:51:54.865358+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:51:54.865358+0000","last_undegraded":"2024-09-13T14:51:54.865358+0000","last_fullsized":"2024-09-13T14:51:54.865358+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":369,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866582+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:51:54.866582+0000","last_peered":"2024-09-13T14:51:54.866582+0000","last_clean":"2024-09-13T14:51:54.866582+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:51:54.866582+0000","last_undegraded":"2024-09-13T14:51:54.866582+0000","last_fullsized":"2024-09-13T14:51:54.866582+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":286,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.865305+0000","last_change":"2024-09-13T14:51:37.794439+0000","last_active":"2024-09-13T14:51:54.865305+0000","last_peered":"2024-09-13T14:51:54.865305+0000","last_clean":"2024-09-13T14:51:54.865305+0000","last_became_active":"2024-09-13T14:51:37.793627+0000","last_became_peered":"2024-09-13T14:51:37.793627+0000","last_unstale":"2024-09-13T14:51:54.865305+0000","last_undegraded":"2024-09-13T14:51:54.865305+0000","last_fullsized":"2024-09-13T14:51:54.865305+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:30.400602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":243,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877440+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:51:53.877440+0000","last_peered":"2024-09-13T14:51:53.877440+0000","last_clean":"2024-09-13T14:51:53.877440+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:51:53.877440+0000","last_undegraded":"2024-09-13T14:51:53.877440+0000","last_fullsized":"2024-09-13T14:51:53.877440+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":230,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876567+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:51:53.876567+0000","last_peered":"2024-09-13T14:51:53.876567+0000","last_clean":"2024-09-13T14:51:53.876567+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:51:53.876567+0000","last_undegraded":"2024-09-13T14:51:53.876567+0000","last_fullsized":"2024-09-13T14:51:53.876567+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":261,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877345+0000","last_change":"2024-09-13T14:51:20.399203+0000","last_active":"2024-09-13T14:51:53.877345+0000","last_peered":"2024-09-13T14:51:53.877345+0000","last_clean":"2024-09-13T14:51:53.877345+0000","last_became_active":"2024-09-13T14:51:20.398824+0000","last_became_peered":"2024-09-13T14:51:20.398824+0000","last_unstale":"2024-09-13T14:51:53.877345+0000","last_undegraded":"2024-09-13T14:51:53.877345+0000","last_fullsized":"2024-09-13T14:51:53.877345+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:35.733324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":215,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876615+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:51:53.876615+0000","last_peered":"2024-09-13T14:51:53.876615+0000","last_clean":"2024-09-13T14:51:53.876615+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:51:53.876615+0000","last_undegraded":"2024-09-13T14:51:53.876615+0000","last_fullsized":"2024-09-13T14:51:53.876615+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":389,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866966+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:51:54.866966+0000","last_peered":"2024-09-13T14:51:54.866966+0000","last_clean":"2024-09-13T14:51:54.866966+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:51:54.866966+0000","last_undegraded":"2024-09-13T14:51:54.866966+0000","last_fullsized":"2024-09-13T14:51:54.866966+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":365,"reported_epoch":261,"state":"active+clean","last_fresh":"2024-09-13T14:51:54.866135+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:51:54.866135+0000","last_peered":"2024-09-13T14:51:54.866135+0000","last_clean":"2024-09-13T14:51:54.866135+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:51:54.866135+0000","last_undegraded":"2024-09-13T14:51:54.866135+0000","last_fullsized":"2024-09-13T14:51:54.866135+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":139,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.877162+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:51:53.877162+0000","last_peered":"2024-09-13T14:51:53.877162+0000","last_clean":"2024-09-13T14:51:53.877162+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:51:53.877162+0000","last_undegraded":"2024-09-13T14:51:53.877162+0000","last_fullsized":"2024-09-13T14:51:53.877162+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":132,"reported_epoch":260,"state":"active+clean","last_fresh":"2024-09-13T14:51:53.876907+0000","last_change":"2024-09-13T14:51:37.788879+0000","last_active":"2024-09-13T14:51:53.876907+0000","last_peered":"2024-09-13T14:51:53.876907+0000","last_clean":"2024-09-13T14:51:53.876907+0000","last_became_active":"2024-09-13T14:51:37.785491+0000","last_became_peered":"2024-09-13T14:51:37.785491+0000","last_unstale":"2024-09-13T14:51:53.876907+0000","last_undegraded":"2024-09-13T14:51:53.876907+0000","last_fullsized":"2024-09-13T14:51:53.876907+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:33.630487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":4140695552,"num_objects":63182,"num_object_clones":0,"num_object_copies":189546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63182,"num_write_kb":4043648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12511543296,"data_stored":12511543296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63182,"ondisk_log_size":63182,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1320,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234384,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2364540,"kb_used_data":1495028,"kb_used_omap":1,"kb_used_meta":869502,"kb_avail":91380612,"statfs":{"total":95995035648,"available":93573746688,"internally_reserved":0,"allocated":1530908672,"data_stored":1529517652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":890370507},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496022,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2824572,"kb_used_data":2003444,"kb_used_omap":1,"kb_used_meta":821118,"kb_avail":90920580,"statfs":{"total":95995035648,"available":93102673920,"internally_reserved":0,"allocated":2051526656,"data_stored":2050136181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":840825287},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757660,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2011824,"kb_used_data":988072,"kb_used_omap":1,"kb_used_meta":1023742,"kb_avail":91733328,"statfs":{"total":95995035648,"available":93934927872,"internally_reserved":0,"allocated":1011785728,"data_stored":1010402998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048312262},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019296,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1855608,"kb_used_data":998384,"kb_used_omap":1,"kb_used_meta":857214,"kb_avail":91889544,"statfs":{"total":95995035648,"available":94094893056,"internally_reserved":0,"allocated":1022345216,"data_stored":1020958278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":877787590},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650508,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2141676,"kb_used_data":1696548,"kb_used_omap":4,"kb_used_meta":445115,"kb_avail":91603476,"statfs":{"total":95995035648,"available":93801959424,"internally_reserved":0,"allocated":1737265152,"data_stored":1735886280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":455798319},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879443,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881780,"kb_used_data":1491372,"kb_used_omap":6,"kb_used_meta":390393,"kb_avail":91863372,"statfs":{"total":95995035648,"available":94068092928,"internally_reserved":0,"allocated":1527164928,"data_stored":1525778480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":399763001},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435460,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3040808,"kb_used_data":2340896,"kb_used_omap":5,"kb_used_meta":699898,"kb_avail":90704344,"statfs":{"total":95995035648,"available":92881248256,"internally_reserved":0,"allocated":2397077504,"data_stored":2395701623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":716695905},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926035,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1385060,"kb_used_data":1064732,"kb_used_omap":5,"kb_used_meta":320314,"kb_avail":92360092,"statfs":{"total":95995035648,"available":94576734208,"internally_reserved":0,"allocated":1090285568,"data_stored":1088914254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":328001706},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1108082688,"data_stored":1108082688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2424438784,"data_stored":2424438784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1544945664,"data_stored":1544945664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1763770368,"data_stored":1763770368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1040252928,"data_stored":1040252928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1020395520,"data_stored":1020395520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2051211264,"data_stored":2051211264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1558446080,"data_stored":1558446080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:10.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_6 pgp_num 16 2024-09-13T14:52:10.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:10.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:52:10.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:10.764 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:52:10.763867+0000 min lat: 0.00116536 max lat: 2.04401 avg lat: 0.00935122 2024-09-13T14:52:10.764 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:52:10.764 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 68427 68411 106.876 81.125 0.0119737 0.00935122 2024-09-13T14:52:10.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:10.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:10.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:10.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:10.936 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:52:11.002 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:11.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:11.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:11.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:11.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:11.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:11.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:10 smithi159 ceph-mon[102519]: from='client.45329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:10 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3452550262' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:10 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[113926]: from='client.45329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3452550262' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[108570]: from='client.45329 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3452550262' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:10 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:11.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:11.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.551 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:11.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:11.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:11.764 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 69726 69710 106.249 81.1875 0.00850535 0.00940736 2024-09-13T14:52:11.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:11.811 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:11.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:11.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:11.938 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:11.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:11.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:12.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[108570]: pgmap v894: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:52:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:52:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:12.044 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[108570]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[113926]: pgmap v894: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:52:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:52:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:12.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:11 smithi042 ceph-mon[113926]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:12.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:12.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:12.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:12.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:11 smithi159 ceph-mon[102519]: pgmap v894: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-13T14:52:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:11 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:52:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:11 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:52:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:11 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:11 smithi159 ceph-mon[102519]: osdmap e262: 8 total, 8 up, 8 in 2024-09-13T14:52:12.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.357 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:12.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:12.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.650 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:12.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:12.684 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:12.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:12.765 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 70950 70934 105.539 76.5 0.00847617 0.00946952 2024-09-13T14:52:12.790 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:12.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:12.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:12.923 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:12.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:12.951 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:13.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.074 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:13.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:13.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.211 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:13.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.353 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:13.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.454 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:13.490 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:13.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:13.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:13.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:13.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:13.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:13.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:13.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:13.765 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 72232 72216 104.947 80.125 0.0106231 0.00952332 2024-09-13T14:52:13.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:13.898 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:13.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:13.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:14.000 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:14.173 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:14.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:13 smithi159 ceph-mon[102519]: pgmap v896: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:52:14.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:13 smithi042 ceph-mon[108570]: pgmap v896: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:52:14.257 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:13 smithi042 ceph-mon[113926]: pgmap v896: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:52:14.275 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:14.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:14.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:14.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:14.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.548 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.574 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:14.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:14.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:14.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:14.765 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 73510 73494 104.377 79.875 0.0156802 0.00957543 2024-09-13T14:52:14.815 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:14.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:14.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:14.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:14.998 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:15.038 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1098671838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:15.038 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1098671838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:15.101 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:15.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:15.211 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1098671838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:15.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:15.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:15.377 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:15.377 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:52:15.377 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T14:52:15.377 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:15.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:15.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:15.519 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:15.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:15.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:15.707 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:15.720 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:52:15.720 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:15.721 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:15.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:15.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:15.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:15.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 74764 74748 103.793 78.375 0.00931308 0.00962856 2024-09-13T14:52:15.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:15.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:15.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:15.968 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:15 smithi042 ceph-mon[108570]: pgmap v897: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:52:15.968 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:15 smithi042 ceph-mon[113926]: pgmap v897: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:52:15.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:15.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:15.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:15.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:15 smithi159 ceph-mon[102519]: pgmap v897: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T14:52:16.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:52:16.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:52:16.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:16.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:16.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:16.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:16.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:16.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:16.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:16.558 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:16.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:16.660 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:16.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:16.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 75882 75866 103.056 69.875 0.0127592 0.00969761 2024-09-13T14:52:16.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:16.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:16.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:16.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:16.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:16.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:16.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:17.063 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:17.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:17.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:17.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:17.241 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:17.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:17.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:17.367 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:17.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:17.494 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:17.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:17.595 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:17.623 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:17.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:17.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:17.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:17.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:17.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 76807 76791 102.093 57.8125 0.0340956 0.00978911 2024-09-13T14:52:17.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:17.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:17.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:17.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:17.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:18.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:18.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:18.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:17 smithi159 ceph-mon[102519]: pgmap v898: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:52:18.281 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:18.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:17 smithi042 ceph-mon[113926]: pgmap v898: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:52:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:17 smithi042 ceph-mon[108570]: pgmap v898: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-13T14:52:18.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:18.463 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:18.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:18.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:18.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:18.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:18.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:18.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:18.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:18.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 77983 77967 101.498 73.5 0.013995 0.00984732 2024-09-13T14:52:18.826 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:18.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:18.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:18.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:18.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:18.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:19.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:19.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:19.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:19.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:19.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:19.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.395 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:19.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:19.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.527 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:19.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:19.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:19.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:19.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:19.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 78914 78898 100.614 58.1875 0.0110095 0.00993372 2024-09-13T14:52:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.795 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:19.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:19.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:19.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:19.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:19.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:20.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:20.057 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:19 smithi042 ceph-mon[108570]: pgmap v899: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:20.058 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:19] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.51.0" 2024-09-13T14:52:20.058 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:19 smithi042 ceph-mon[113926]: pgmap v899: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:20.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:20.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:20.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:20.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:19 smithi159 ceph-mon[102519]: pgmap v899: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:20.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:20.363 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:20.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:20.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:20.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:20.640 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:20.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:20.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:20.768 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 80030 80014 99.9963 69.75 0.0100555 0.00999491 2024-09-13T14:52:20.781 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:20.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:20.786 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:52:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:52:20.786 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:52:20.786 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:20.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:20.922 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:20.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:21.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:21.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:21.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:21.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:21.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:21.335 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:21.417 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:21.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:21.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:21.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:21.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:21.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.751 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:21.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:21.769 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 80874 80858 99.0681 52.75 0.0107733 0.0100892 2024-09-13T14:52:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: pgmap v900: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-09-13T14:52:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"} : dispatch 2024-09-13T14:52:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2540227859' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:21.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: pgmap v900: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-09-13T14:52:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"} : dispatch 2024-09-13T14:52:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2540227859' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:21.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:21 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:21.852 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:21.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:21.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:21.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:21.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:21.993 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 11 pgp_num to 16 2024-09-13T14:52:22.024 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: pgmap v900: 99 pgs: 99 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]} : dispatch 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"} : dispatch 2024-09-13T14:52:22.081 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2540227859' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:22.082 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:21 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:22.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:22.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:22.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:22.408 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:22.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:22.592 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:22.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:22.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:22.769 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 82094 82078 98.6292 76.25 0.0110282 0.0101334 2024-09-13T14:52:22.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:52:22.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:22.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:22.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:22.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:22.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:22.978 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:23.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:23.104 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:23.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:23.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"}]': finished 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:23.316 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:23.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"}]': finished 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-09-13T14:52:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "3.a"}]': finished 2024-09-13T14:52:23.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:23.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: osdmap e263: 8 total, 8 up, 8 in 2024-09-13T14:52:23.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:23.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:23.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:23.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:23.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:23.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:23.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:23.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:23.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 83191 83175 98.0617 68.5625 0.0114891 0.0101922 2024-09-13T14:52:23.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:23.877 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:23.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:23.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:23.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:24.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:24.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:24.115 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.159 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.206 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:24 smithi042 ceph-mon[108570]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:52:24.207 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:24 smithi042 ceph-mon[108570]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:24.207 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:24 smithi042 ceph-mon[113926]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:52:24.207 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:24 smithi042 ceph-mon[113926]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:24.261 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:24.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:24.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:24.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:24 smithi159 ceph-mon[102519]: pgmap v902: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-09-13T14:52:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:24 smithi159 ceph-mon[102519]: osdmap e264: 8 total, 8 up, 8 in 2024-09-13T14:52:24.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.593 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:24.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:24.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:24.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:24.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:24.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 84342 84326 97.5779 71.9375 0.0094857 0.0102425 2024-09-13T14:52:24.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:24.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:24.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:24.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:24.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:24.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:25.007 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:52:25.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:25.133 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:25.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:25.192 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3240280224' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.192 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:25 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64447 10.11 10.84e2efb1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:24.434720+0000", 2024-09-13T14:52:25.222 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 300.78409826299998, 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89696834800000003, 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64447 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434720+0000", 2024-09-13T14:52:25.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434720+0000", 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434718+0000", 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434850+0000", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013280200000000001 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434871+0000", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0491000000000001e-05 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434881+0000", 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0118e-05 2024-09-13T14:52:25.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434935+0000", 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4159000000000001e-05 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.434942+0000", 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3759999999999996e-06 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331678+0000", 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89673599400000004 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331688+0000", 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0604e-05 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64468 10.11 10.fa6c1831 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-13T14:52:25.227 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:24.553508+0000", 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 300.665310148, 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77820728800000005, 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64468 2024-09-13T14:52:25.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553508+0000", 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553508+0000", 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553506+0000", 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553560+0000", 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4160999999999999e-05 2024-09-13T14:52:25.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553579+0000", 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8607999999999999e-05 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553587+0000", 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4519999999999997e-06 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553651+0000", 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3034000000000007e-05 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.553659+0000", 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7900000000000005e-06 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331709+0000", 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77804934800000003 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331715+0000", 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6370000000000002e-06 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.233 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:64477 10.11 10.8993d931 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e197)", 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:24.557555+0000", 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 300.66126291199998, 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77418983699999999, 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 64477 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557555+0000", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557555+0000", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557553+0000", 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557611+0000", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7572999999999999e-05 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557624+0000", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3278999999999999e-05 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557631+0000", 2024-09-13T14:52:25.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1860000000000003e-06 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557681+0000", 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9134000000000001e-05 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:24.557688+0000", 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4569999999999999e-06 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331738+0000", 2024-09-13T14:52:25.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77405008500000005 2024-09-13T14:52:25.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:25.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:25.331745+0000", 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9739999999999996e-06 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116270 10.11 10:8d673ec4:::benchmark_data_smithi042_191918_object116269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:11.937780+0000", 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.281037954, 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.032192639, 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:52:25.240 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116270 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937780+0000", 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937780+0000", 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937764+0000", 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999838 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937844+0000", 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9762e-05 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937869+0000", 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4839999999999999e-05 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.937876+0000", 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3420000000000002e-06 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.940223+0000", 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002347292 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:11.940232+0000", 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3070000000000003e-06 2024-09-13T14:52:25.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.969880+0000", 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0296488349999999 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.969973+0000", 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2421000000000002e-05 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116345 10.11 10:8d969239:::benchmark_data_smithi042_191918_object116344:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.058002+0000", 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.160815966, 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.92289684599999999, 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116345 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058002+0000", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058002+0000", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058000+0000", 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T14:52:25.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058050+0000", 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0124000000000003e-05 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058071+0000", 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0404e-05 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058079+0000", 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4500000000000004e-06 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058195+0000", 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000115542 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.058205+0000", 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0378e-05 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.970285+0000", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91208003000000004 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.970425+0000", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000140195 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.970585+0000", 2024-09-13T14:52:25.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000159977 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.976726+0000", 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061406109999999998 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.976801+0000", 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5043000000000006e-05 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.980790+0000", 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039894409999999998 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.980841+0000", 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1221000000000003e-05 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.980899+0000", 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7596000000000002e-05 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.252 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116368 10.11 10:898e4f45:::benchmark_data_smithi042_191918_object116367:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.071526+0000", 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.147292134, 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90956498299999999, 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.253 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116368 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071526+0000", 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071526+0000", 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071524+0000", 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071585+0000", 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1362e-05 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071601+0000", 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.524e-05 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.071608+0000", 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0169999999999999e-06 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.072399+0000", 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00079098899999999997 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.072408+0000", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1039999999999996e-06 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.972309+0000", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89990116899999995 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.972453+0000", 2024-09-13T14:52:25.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014366699999999999 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.972594+0000", 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014139600000000001 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.976791+0000", 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041972600000000004 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.978874+0000", 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0020827419999999998 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981013+0000", 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002139307 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981059+0000", 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5763999999999998e-05 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981091+0000", 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1850000000000002e-05 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116411 10.11 10:88d2e388:::benchmark_data_smithi042_191918_object116410:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.108703+0000", 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.110114786, 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.87254129400000002, 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116411 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108703+0000", 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108703+0000", 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108702+0000", 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108778+0000", 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5327000000000006e-05 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108790+0000", 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2012e-05 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.108796+0000", 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7850000000000003e-06 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.110257+0000", 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0014611069999999999 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.110266+0000", 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8079999999999994e-06 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.974603+0000", 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.86433701299999999 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.974753+0000", 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014979899999999999 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.974952+0000", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00019884699999999999 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.980912+0000", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0059596889999999998 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.980965+0000", 2024-09-13T14:52:25.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3650000000000003e-05 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981143+0000", 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017727899999999999 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981207+0000", 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4218000000000001e-05 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981245+0000", 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7851000000000001e-05 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116416 10.11 10:887da574:::benchmark_data_smithi042_191918_object116415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.112834+0000", 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.10598369300001, 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.87243828199999995, 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.268 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116416 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112834+0000", 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112834+0000", 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112833+0000", 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112901+0000", 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7873000000000006e-05 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112913+0000", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1813e-05 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.112919+0000", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8990000000000004e-06 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.113008+0000", 2024-09-13T14:52:25.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8300999999999994e-05 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.113017+0000", 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5389999999999996e-06 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.976826+0000", 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.86380905799999996 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.976974+0000", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014730599999999999 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.977117+0000", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000143227 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981103+0000", 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039856650000000002 2024-09-13T14:52:25.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981312+0000", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000209032 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985190+0000", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038785320000000001 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985237+0000", 2024-09-13T14:52:25.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7207999999999999e-05 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985273+0000", 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5413999999999997e-05 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116430 10.11 10:8e2477f8:::benchmark_data_smithi042_191918_object116429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.127312+0000", 2024-09-13T14:52:25.275 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.091506162, 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85808768800000002, 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116430 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127312+0000", 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127312+0000", 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127310+0000", 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127365+0000", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4667999999999999e-05 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127381+0000", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5922000000000001e-05 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127389+0000", 2024-09-13T14:52:25.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4179999999999997e-06 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127470+0000", 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0458999999999997e-05 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.127480+0000", 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0069e-05 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.978920+0000", 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85143996600000005 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.979042+0000", 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000122383 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.979130+0000", 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8016000000000006e-05 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981109+0000", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001978783 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985313+0000", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042038279999999997 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985339+0000", 2024-09-13T14:52:25.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6342999999999998e-05 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985372+0000", 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2929000000000003e-05 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985400+0000", 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7429000000000001e-05 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116452 10.11 10:8eca6023:::benchmark_data_smithi042_191918_object116451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.145126+0000", 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.07369199600001, 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85083672600000004, 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.283 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.285 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.285 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.285 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116452 2024-09-13T14:52:25.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.285 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145126+0000", 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145126+0000", 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145125+0000", 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145206+0000", 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1012000000000003e-05 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145222+0000", 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6248000000000001e-05 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145230+0000", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5989999999999996e-06 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145284+0000", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4317999999999997e-05 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145294+0000", 2024-09-13T14:52:25.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0094e-05 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981353+0000", 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.836058827 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981492+0000", 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013943500000000001 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.981693+0000", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000200619 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985430+0000", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037366539999999998 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985467+0000", 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7870999999999997e-05 2024-09-13T14:52:25.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.995737+0000", 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010270013999999999 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.995918+0000", 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018100700000000001 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.995963+0000", 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4433e-05 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116453 10.11 10:8ba4e5e9:::benchmark_data_smithi042_191918_object116452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.145378+0000", 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.07344053899999, 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85075079899999995, 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.292 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116453 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145378+0000", 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145378+0000", 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145377+0000", 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145509+0000", 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000132626 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145533+0000", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3078000000000002e-05 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145540+0000", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7149999999999998e-06 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145584+0000", 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3784e-05 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.145593+0000", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7090000000000008e-06 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.983446+0000", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.837853132 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.983610+0000", 2024-09-13T14:52:25.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000164072 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.983746+0000", 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013587500000000001 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989471+0000", 2024-09-13T14:52:25.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057249349999999996 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989498+0000", 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7665999999999999e-05 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.995971+0000", 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0064728279999999999 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996085+0000", 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000113616 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996128+0000", 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3535000000000002e-05 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.299 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116534 10.11 10:8ccae711:::benchmark_data_smithi042_191918_object116533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.245215+0000", 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.973603443, 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.75808245100000005, 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.300 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116534 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245215+0000", 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245215+0000", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245213+0000", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245271+0000", 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7367999999999997e-05 2024-09-13T14:52:25.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245287+0000", 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6528000000000002e-05 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245296+0000", 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3340000000000002e-06 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245353+0000", 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7667000000000002e-05 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.245364+0000", 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0845999999999999e-05 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985488+0000", 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.74012389099999998 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985623+0000", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000134855 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.985759+0000", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013646900000000001 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989532+0000", 2024-09-13T14:52:25.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037724669999999998 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996139+0000", 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066076729999999997 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003217+0000", 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0070779379999999998 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003263+0000", 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5348e-05 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003297+0000", 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4375999999999997e-05 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116548 10.11 10:8d1587f2:::benchmark_data_smithi042_191918_object116547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.307 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.258940+0000", 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.959878463, 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.74454047000000001, 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116548 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.308 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.258940+0000", 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.258940+0000", 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.258938+0000", 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.259049+0000", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011089700000000001 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.259061+0000", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2238999999999999e-05 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.259068+0000", 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4999999999999996e-06 2024-09-13T14:52:25.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.259129+0000", 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1743999999999997e-05 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.259139+0000", 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7829999999999993e-06 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.987478+0000", 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72833859499999998 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.987611+0000", 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000132877 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.987732+0000", 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121032 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.995730+0000", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0079985779999999992 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996144+0000", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041426100000000002 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003418+0000", 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0072731619999999997 2024-09-13T14:52:25.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003452+0000", 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4109999999999997e-05 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003480+0000", 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8379999999999999e-05 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116591 10.11 10:8997fe6c:::benchmark_data_smithi042_191918_object116590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.348909+0000", 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.86990930299999, 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65467737000000004, 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116591 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.315 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348909+0000", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348909+0000", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348907+0000", 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348963+0000", 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6056e-05 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348977+0000", 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3494e-05 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.348984+0000", 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8730000000000001e-06 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.349044+0000", 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0126e-05 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.349053+0000", 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0019999999999995e-06 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989552+0000", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64049875999999994 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989670+0000", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117934 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.989791+0000", 2024-09-13T14:52:25.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121258 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996149+0000", 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0063581599999999999 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003337+0000", 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071881610000000002 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003520+0000", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018300200000000001 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003554+0000", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3614e-05 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003586+0000", 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2502999999999997e-05 2024-09-13T14:52:25.321 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116594 10.11 10:88aa7d17:::benchmark_data_smithi042_191918_object116593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.352310+0000", 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.866507923, 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65136538600000005, 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.323 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116594 2024-09-13T14:52:25.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.323 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352310+0000", 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352310+0000", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352309+0000", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352418+0000", 2024-09-13T14:52:25.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010959399999999999 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352433+0000", 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4153e-05 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352440+0000", 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0360000000000001e-06 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352494+0000", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4543000000000003e-05 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.352505+0000", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0621e-05 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.991591+0000", 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.63908602400000003 2024-09-13T14:52:25.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.991708+0000", 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011709000000000001 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.991823+0000", 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000115468 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003177+0000", 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.011353569000000001 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003364+0000", 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018708400000000001 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003618+0000", 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00025348800000000001 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003649+0000", 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1461999999999998e-05 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003676+0000", 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6605e-05 2024-09-13T14:52:25.330 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116597 10.11 10:88f71793:::benchmark_data_smithi042_191918_object116596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.353242+0000", 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.86557631599999, 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65052153999999995, 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.331 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116597 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353242+0000", 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353242+0000", 2024-09-13T14:52:25.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353241+0000", 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353314+0000", 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2570000000000005e-05 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353326+0000", 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2556e-05 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353334+0000", 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5349999999999999e-06 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353379+0000", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5747000000000001e-05 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.353386+0000", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.922e-06 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.993651+0000", 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64026427600000002 2024-09-13T14:52:25.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.993776+0000", 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000125578 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.993903+0000", 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012716800000000001 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003182+0000", 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0092788820000000004 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003391+0000", 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000209006 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003705+0000", 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00031401999999999998 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003737+0000", 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1251000000000003e-05 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003763+0000", 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6821999999999998e-05 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.338 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116610 10.11 10:897c792c:::benchmark_data_smithi042_191918_object116609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.370804+0000", 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.848014143, 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.641989319, 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116610 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370804+0000", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370804+0000", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370803+0000", 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370839+0000", 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5846000000000002e-05 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370855+0000", 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5985e-05 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370863+0000", 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1270000000000003e-06 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370928+0000", 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5671000000000002e-05 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.370937+0000", 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9970000000000004e-06 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996180+0000", 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.62524281699999995 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996308+0000", 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012777599999999999 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.996437+0000", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012949000000000001 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003307+0000", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0068694100000000003 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003793+0000", 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048622800000000001 2024-09-13T14:52:25.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.012714+0000", 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.008921221 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.012759+0000", 2024-09-13T14:52:25.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5003999999999998e-05 2024-09-13T14:52:25.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:25.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.012793+0000", 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3998999999999998e-05 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116611 10.11 10:8b1147ee:::benchmark_data_smithi042_191918_object116610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.371639+0000", 2024-09-13T14:52:25.347 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.847179431, 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64141564900000003, 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116611 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371639+0000", 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371639+0000", 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371638+0000", 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371693+0000", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4867999999999997e-05 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371704+0000", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0691e-05 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371709+0000", 2024-09-13T14:52:25.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.074e-06 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371744+0000", 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4977000000000001e-05 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.371751+0000", 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1389999999999998e-06 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.998321+0000", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.62656993299999997 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.998481+0000", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016039900000000001 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.998629+0000", 2024-09-13T14:52:25.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014759699999999999 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.003821+0000", 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0051924470000000002 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.012805+0000", 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0089841230000000001 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.012992+0000", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000186696 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.013026+0000", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3701999999999997e-05 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.013054+0000", 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8753999999999999e-05 2024-09-13T14:52:25.354 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:116616 10.11 10:8f01855a:::benchmark_data_smithi042_191918_object116615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e208)", 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:12.402873+0000", 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.81594541999999, 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.61875638700000002, 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.355 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 116616 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402873+0000", 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402873+0000", 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402871+0000", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402932+0000", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0192999999999998e-05 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402943+0000", 2024-09-13T14:52:25.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1538e-05 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402949+0000", 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7540000000000003e-06 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.402996+0000", 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7182999999999999e-05 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:52:25.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:12.403003+0000", 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9859999999999997e-06 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.004342+0000", 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.60133928999999997 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.004801+0000", 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00045825 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 6,7", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.008654+0000", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038537770000000001 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.013184+0000", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045299900000000002 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.019877+0000", 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066927389999999996 2024-09-13T14:52:25.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.021483+0000", 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0016058240000000001 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.021590+0000", 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106989 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:13.021629+0000", 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9175999999999999e-05 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:118886 10.9 10:905d0335:::benchmark_data_smithi042_191918_object118885:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:15.270740+0000", 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 249.94807766, 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.48740159199999999, 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:25.362 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 118886 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270740+0000", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270740+0000", 2024-09-13T14:52:25.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270740+0000", 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270764+0000", 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3966000000000001e-05 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270775+0000", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0842999999999999e-05 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.270781+0000", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0039999999999996e-06 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.272972+0000", 2024-09-13T14:52:25.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002191082 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.273099+0000", 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012751 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.273271+0000", 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017214800000000001 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.275626+0000", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002354465 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.275879+0000", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00025296599999999998 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.758047+0000", 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.482168596 2024-09-13T14:52:25.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.758105+0000", 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7259999999999997e-05 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:15.758142+0000", 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7395999999999999e-05 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.368 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.369 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:25.369 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:25.369 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:25.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3240280224' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:25 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.370 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3240280224' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.370 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:25 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:52:25.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:25.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:25.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:25.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:25.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:25.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:25.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:25.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 85447 85431 97.0593 69.0625 0.0121231 0.0102976 2024-09-13T14:52:25.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:25.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:25.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:25.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:25.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.046 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:26.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:26.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:26.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:52:26.296 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:26.309 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:26 smithi159 ceph-mon[102519]: pgmap v904: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:26.309 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:26 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:26.309 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:26 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:26.309 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:26 smithi159 ceph-mon[102519]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:26.335 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[113926]: pgmap v904: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[113926]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[108570]: pgmap v904: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:52:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:26 smithi042 ceph-mon[108570]: osdmap e265: 8 total, 8 up, 8 in 2024-09-13T14:52:26.351 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:26.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:26.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:26.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.584 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:26.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:26.740 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:26.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 86610 86594 96.6241 72.6875 0.0130917 0.0103443 2024-09-13T14:52:26.771 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:26.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:26.873 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:26.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:26.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:27.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:27.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:27.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:27.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:27.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.736 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 87722 87706 96.1482 69.5 0.0274352 0.0103946 2024-09-13T14:52:27.772 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:27.878 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:27.915 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:27.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:27.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:28.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:28.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:28.081 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:52:28.081 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:28.081 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_6 2024-09-13T14:52:28.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:52:28.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.169 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:28 smithi159 ceph-mon[102519]: pgmap v906: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:52:28.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.285 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:28 smithi042 ceph-mon[113926]: pgmap v906: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:52:28.286 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:28 smithi042 ceph-mon[108570]: pgmap v906: 99 pgs: 99 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-13T14:52:28.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:28.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:28.528 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:28.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:28.663 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:28.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:28.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:28.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 88699 88683 95.5432 61.0625 0.00306606 0.0104577 2024-09-13T14:52:28.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:28.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:28.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:28.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:52:28.944 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:28.961 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:28.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:28.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:28.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:29.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:29.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.115 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:29.216 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:29.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:29.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:29.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:29.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:29.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:29.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:29.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:29.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 89673 89657 94.9553 60.875 0.0123696 0.0105263 2024-09-13T14:52:29.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:29.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:29.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:29.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:30.036 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:29] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-09-13T14:52:30.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:30.232 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:30 smithi159 ceph-mon[102519]: pgmap v907: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2024-09-13T14:52:30.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:30 smithi042 ceph-mon[113926]: pgmap v907: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2024-09-13T14:52:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:30 smithi042 ceph-mon[108570]: pgmap v907: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s 2024-09-13T14:52:30.366 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:30.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:30.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:30.535 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:30.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:30.619 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45314.0:90523 11.1 11:87fab597:::benchmark_data_smithi042_223320_object90522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:30.529386+0000", 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.089026543999999999, 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.089057485000000006, 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:30.620 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45314", 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 90523 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529386+0000", 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:30.621 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529386+0000", 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529385+0000", 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529426+0000", 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1465999999999999e-05 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.622 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529443+0000", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7368e-05 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529452+0000", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.225e-06 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529504+0000", 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2544999999999999e-05 2024-09-13T14:52:30.623 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529674+0000", 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000169502 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.624 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.529879+0000", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000205412 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.533201+0000", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033218599999999998 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.533494+0000", 2024-09-13T14:52:30.625 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00029349500000000001 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45314.0:90531 11.1 11:86693d34:::benchmark_data_smithi042_223320_object90530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:30.536957+0000", 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.081455062999999994, 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.081543161000000003, 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45314", 2024-09-13T14:52:30.626 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 90531 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.536957+0000", 2024-09-13T14:52:30.627 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.536957+0000", 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:30.628 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.536956+0000", 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537024+0000", 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8095000000000004e-05 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537039+0000", 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5165e-05 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.629 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537044+0000", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0830000000000003e-06 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537076+0000", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1680000000000002e-05 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537191+0000", 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000114468 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.630 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,3", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.537312+0000", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012143500000000001 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.539660+0000", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023480900000000002 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:30.539885+0000", 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000225322 2024-09-13T14:52:30.631 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 2 2024-09-13T14:52:30.632 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:30.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:30.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:30.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:52:30.770446+0000 min lat: 0.00116536 max lat: 2.04401 avg lat: 0.0105828 2024-09-13T14:52:30.771 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:52:30.771 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 90701 90685 94.4435 64.25 0.00598458 0.0105828 2024-09-13T14:52:30.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:30.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:30.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:30.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:31.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:31.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:31.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:31.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:31.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:31.368 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:31.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:52:31.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:31.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:31.636 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:31.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:31.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:31.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:31.771 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 91480 91464 93.6934 48.6875 0.241736 0.010642 2024-09-13T14:52:31.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:31.831 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:31.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:31.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:31.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:32.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:32.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:32.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:32.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:32.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:32.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:32 smithi042 ceph-mon[113926]: pgmap v908: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2024-09-13T14:52:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:32 smithi042 ceph-mon[108570]: pgmap v908: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2024-09-13T14:52:32.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:32.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:32.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:32 smithi159 ceph-mon[102519]: pgmap v908: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s 2024-09-13T14:52:32.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:32.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:32.661 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:32.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:32.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:32.770 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 92613 92597 93.3243 70.8125 0.0272565 0.0107092 2024-09-13T14:52:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.811 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:32.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:32.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:32.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:32.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:33.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:33.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:33.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:33.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:33.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:33.365 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:33.467 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.531 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:33.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.632 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.678 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:33.678 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:52:33.678 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:52:33.678 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:52:33.678 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:33.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.745 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:52:33.745 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:33.749 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:52:33.749 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:33.772 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 93614 93598 92.8349 62.5625 0.0174645 0.0107662 2024-09-13T14:52:33.784 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:52:33.784 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:33.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:33.824 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:52:33.824 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:33.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:33.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:52:33.858 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:33.892 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:52:33.892 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:33.932 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:52:33.932 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:33.935 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:33.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:33.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:33.984 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:52:33.984 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:34.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.123 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:34.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:34.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:34.399 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:52:34.401 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:52:34.407 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: pgmap v909: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3455276146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:34.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:34.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:34.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: pgmap v909: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3455276146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: pgmap v909: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3455276146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:34.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:52:34.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:52:34.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:34.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:34.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:34.772 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 94538 94522 92.2866 57.75 0.0184795 0.0108305 2024-09-13T14:52:34.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:34.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:34.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:34.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:52:34.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:34.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:34.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:35.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.105 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:35.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:35.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:35.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:35.279 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:35.347 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:35.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.391 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:35.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:35.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:35.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:35.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:35.771 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 95646 95630 91.9321 69.25 0.00808128 0.0108724 2024-09-13T14:52:35.800 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":909,"stamp":"2024-09-13T14:52:32.390662+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5931538405,"num_objects":90525,"num_object_clones":0,"num_object_copies":271575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90563,"num_write_kb":5791969,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1208,"num_bytes_recovered":77333800,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92197,"ondisk_log_size":92197,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22789708,"kb_used_data":17290188,"kb_used_omap":28,"kb_used_meta":5499427,"kb_avail":727171508,"statfs":{"total":767960285184,"available":744623624192,"internally_reserved":0,"allocated":17705152512,"data_stored":17693968722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":5631413649},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,6,7,11,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":688324608,"num_objects":10503,"num_object_clones":0,"num_object_copies":31509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10503,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10503,"num_write_kb":672192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.403939"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":352,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012842+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:52:25.012842+0000","last_peered":"2024-09-13T14:52:25.012842+0000","last_clean":"2024-09-13T14:52:25.012842+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:52:25.012842+0000","last_undegraded":"2024-09-13T14:52:25.012842+0000","last_fullsized":"2024-09-13T14:52:25.012842+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":328,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005998+0000","last_change":"2024-09-13T14:43:35.907052+0000","last_active":"2024-09-13T14:52:25.005998+0000","last_peered":"2024-09-13T14:52:25.005998+0000","last_clean":"2024-09-13T14:52:25.005998+0000","last_became_active":"2024-09-13T14:43:35.906588+0000","last_became_peered":"2024-09-13T14:43:35.906588+0000","last_unstale":"2024-09-13T14:52:25.005998+0000","last_undegraded":"2024-09-13T14:52:25.005998+0000","last_fullsized":"2024-09-13T14:52:25.005998+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:56.521107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003166+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:52:23.003166+0000","last_peered":"2024-09-13T14:52:23.003166+0000","last_clean":"2024-09-13T14:52:23.003166+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:52:23.003166+0000","last_undegraded":"2024-09-13T14:52:23.003166+0000","last_fullsized":"2024-09-13T14:52:23.003166+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.010333+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:52:25.010333+0000","last_peered":"2024-09-13T14:52:25.010333+0000","last_clean":"2024-09-13T14:52:25.010333+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:52:25.010333+0000","last_undegraded":"2024-09-13T14:52:25.010333+0000","last_fullsized":"2024-09-13T14:52:25.010333+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.010349+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:52:25.010349+0000","last_peered":"2024-09-13T14:52:25.010349+0000","last_clean":"2024-09-13T14:52:25.010349+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:52:25.010349+0000","last_undegraded":"2024-09-13T14:52:25.010349+0000","last_fullsized":"2024-09-13T14:52:25.010349+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":123,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990044+0000","last_change":"2024-09-13T14:43:35.907589+0000","last_active":"2024-09-13T14:52:22.990044+0000","last_peered":"2024-09-13T14:52:22.990044+0000","last_clean":"2024-09-13T14:52:22.990044+0000","last_became_active":"2024-09-13T14:43:35.907254+0000","last_became_peered":"2024-09-13T14:43:35.907254+0000","last_unstale":"2024-09-13T14:52:22.990044+0000","last_undegraded":"2024-09-13T14:52:22.990044+0000","last_fullsized":"2024-09-13T14:52:22.990044+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:13.369649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":377,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012755+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:52:25.012755+0000","last_peered":"2024-09-13T14:52:25.012755+0000","last_clean":"2024-09-13T14:52:25.012755+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:52:25.012755+0000","last_undegraded":"2024-09-13T14:52:25.012755+0000","last_fullsized":"2024-09-13T14:52:25.012755+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":299,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992713+0000","last_change":"2024-09-13T14:51:37.799522+0000","last_active":"2024-09-13T14:52:22.992713+0000","last_peered":"2024-09-13T14:52:22.992713+0000","last_clean":"2024-09-13T14:52:22.992713+0000","last_became_active":"2024-09-13T14:51:37.789544+0000","last_became_peered":"2024-09-13T14:51:37.789544+0000","last_unstale":"2024-09-13T14:52:22.992713+0000","last_undegraded":"2024-09-13T14:52:22.992713+0000","last_fullsized":"2024-09-13T14:52:22.992713+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:27.557594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"265'5646","reported_seq":5676,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.378860+0000","last_change":"2024-09-13T14:51:16.446237+0000","last_active":"2024-09-13T14:52:31.378860+0000","last_peered":"2024-09-13T14:52:31.378860+0000","last_clean":"2024-09-13T14:52:31.378860+0000","last_became_active":"2024-09-13T14:51:16.445868+0000","last_became_peered":"2024-09-13T14:51:16.445868+0000","last_unstale":"2024-09-13T14:52:31.378860+0000","last_undegraded":"2024-09-13T14:52:31.378860+0000","last_fullsized":"2024-09-13T14:52:31.378860+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:14.921393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370016256,"num_objects":5646,"num_object_clones":0,"num_object_copies":16938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5646,"num_write_kb":361344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":174,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.991826+0000","last_change":"2024-09-13T14:51:37.793374+0000","last_active":"2024-09-13T14:52:22.991826+0000","last_peered":"2024-09-13T14:52:22.991826+0000","last_clean":"2024-09-13T14:52:22.991826+0000","last_became_active":"2024-09-13T14:51:37.788764+0000","last_became_peered":"2024-09-13T14:51:37.788764+0000","last_unstale":"2024-09-13T14:52:22.991826+0000","last_undegraded":"2024-09-13T14:52:22.991826+0000","last_fullsized":"2024-09-13T14:52:22.991826+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:44.417775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002700+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:52:23.002700+0000","last_peered":"2024-09-13T14:52:23.002700+0000","last_clean":"2024-09-13T14:52:23.002700+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:52:23.002700+0000","last_undegraded":"2024-09-13T14:52:23.002700+0000","last_fullsized":"2024-09-13T14:52:23.002700+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"265'5665","reported_seq":5711,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:27.838216+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:52:27.838216+0000","last_peered":"2024-09-13T14:52:27.838216+0000","last_clean":"2024-09-13T14:52:27.838216+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:52:27.838216+0000","last_undegraded":"2024-09-13T14:52:27.838216+0000","last_fullsized":"2024-09-13T14:52:27.838216+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5665,"log_dups_size":0,"ondisk_log_size":5665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:23:00.227060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371261440,"num_objects":5665,"num_object_clones":0,"num_object_copies":16995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5665,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":135,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992058+0000","last_change":"2024-09-13T14:43:35.905294+0000","last_active":"2024-09-13T14:52:22.992058+0000","last_peered":"2024-09-13T14:52:22.992058+0000","last_clean":"2024-09-13T14:52:22.992058+0000","last_became_active":"2024-09-13T14:43:35.904851+0000","last_became_peered":"2024-09-13T14:43:35.904851+0000","last_unstale":"2024-09-13T14:52:22.992058+0000","last_undegraded":"2024-09-13T14:52:22.992058+0000","last_fullsized":"2024-09-13T14:52:22.992058+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:21.591225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":258,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003437+0000","last_change":"2024-09-13T14:52:23.002072+0000","last_active":"2024-09-13T14:52:23.003437+0000","last_peered":"2024-09-13T14:52:23.003437+0000","last_clean":"2024-09-13T14:52:23.003437+0000","last_became_active":"2024-09-13T14:52:23.001698+0000","last_became_peered":"2024-09-13T14:52:23.001698+0000","last_unstale":"2024-09-13T14:52:23.003437+0000","last_undegraded":"2024-09-13T14:52:23.003437+0000","last_fullsized":"2024-09-13T14:52:23.003437+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:47.065512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"265'5618","reported_seq":5648,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.380759+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:52:31.380759+0000","last_peered":"2024-09-13T14:52:31.380759+0000","last_clean":"2024-09-13T14:52:31.380759+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:52:31.380759+0000","last_undegraded":"2024-09-13T14:52:31.380759+0000","last_fullsized":"2024-09-13T14:52:31.380759+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368181248,"num_objects":5618,"num_object_clones":0,"num_object_copies":16854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5618,"num_write_kb":359552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":135,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003096+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:52:23.003096+0000","last_peered":"2024-09-13T14:52:23.003096+0000","last_clean":"2024-09-13T14:52:23.003096+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:52:23.003096+0000","last_undegraded":"2024-09-13T14:52:23.003096+0000","last_fullsized":"2024-09-13T14:52:23.003096+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992336+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:52:22.992336+0000","last_peered":"2024-09-13T14:52:22.992336+0000","last_clean":"2024-09-13T14:52:22.992336+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:52:22.992336+0000","last_undegraded":"2024-09-13T14:52:22.992336+0000","last_fullsized":"2024-09-13T14:52:22.992336+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"265'5749","reported_seq":5779,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.395205+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:52:31.395205+0000","last_peered":"2024-09-13T14:52:31.395205+0000","last_clean":"2024-09-13T14:52:31.395205+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:52:31.395205+0000","last_undegraded":"2024-09-13T14:52:31.395205+0000","last_fullsized":"2024-09-13T14:52:31.395205+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376766464,"num_objects":5749,"num_object_clones":0,"num_object_copies":17247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002777+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:52:23.002777+0000","last_peered":"2024-09-13T14:52:23.002777+0000","last_clean":"2024-09-13T14:52:23.002777+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:52:23.002777+0000","last_undegraded":"2024-09-13T14:52:23.002777+0000","last_fullsized":"2024-09-13T14:52:23.002777+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012998+0000","last_change":"2024-09-13T14:51:20.405103+0000","last_active":"2024-09-13T14:52:25.012998+0000","last_peered":"2024-09-13T14:52:25.012998+0000","last_clean":"2024-09-13T14:52:25.012998+0000","last_became_active":"2024-09-13T14:51:20.404344+0000","last_became_peered":"2024-09-13T14:51:20.404344+0000","last_unstale":"2024-09-13T14:52:25.012998+0000","last_undegraded":"2024-09-13T14:52:25.012998+0000","last_fullsized":"2024-09-13T14:52:25.012998+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:05.332557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"265'5721","reported_seq":5751,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.401968+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:52:31.401968+0000","last_peered":"2024-09-13T14:52:31.401968+0000","last_clean":"2024-09-13T14:52:31.401968+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:52:31.401968+0000","last_undegraded":"2024-09-13T14:52:31.401968+0000","last_fullsized":"2024-09-13T14:52:31.401968+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5721,"log_dups_size":0,"ondisk_log_size":5721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374931456,"num_objects":5721,"num_object_clones":0,"num_object_copies":17163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5721,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":702,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012549+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:52:25.012549+0000","last_peered":"2024-09-13T14:52:25.012549+0000","last_clean":"2024-09-13T14:52:25.012549+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:52:25.012549+0000","last_undegraded":"2024-09-13T14:52:25.012549+0000","last_fullsized":"2024-09-13T14:52:25.012549+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":373,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016397+0000","last_change":"2024-09-13T14:51:37.799188+0000","last_active":"2024-09-13T14:52:25.016397+0000","last_peered":"2024-09-13T14:52:25.016397+0000","last_clean":"2024-09-13T14:52:25.016397+0000","last_became_active":"2024-09-13T14:51:37.792663+0000","last_became_peered":"2024-09-13T14:51:37.792663+0000","last_unstale":"2024-09-13T14:52:25.016397+0000","last_undegraded":"2024-09-13T14:52:25.016397+0000","last_fullsized":"2024-09-13T14:52:25.016397+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:04:04.567415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990058+0000","last_change":"2024-09-13T14:40:37.187426+0000","last_active":"2024-09-13T14:52:22.990058+0000","last_peered":"2024-09-13T14:52:22.990058+0000","last_clean":"2024-09-13T14:52:22.990058+0000","last_became_active":"2024-09-13T14:40:37.187117+0000","last_became_peered":"2024-09-13T14:40:37.187117+0000","last_unstale":"2024-09-13T14:52:22.990058+0000","last_undegraded":"2024-09-13T14:52:22.990058+0000","last_fullsized":"2024-09-13T14:52:22.990058+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:10:18.250820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.2","version":"265'5610","reported_seq":5671,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.126682+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:52:31.126682+0000","last_peered":"2024-09-13T14:52:31.126682+0000","last_clean":"2024-09-13T14:52:31.126682+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:52:31.126682+0000","last_undegraded":"2024-09-13T14:52:31.126682+0000","last_fullsized":"2024-09-13T14:52:31.126682+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5610,"log_dups_size":0,"ondisk_log_size":5610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:13:34.283673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367656960,"num_objects":5610,"num_object_clones":0,"num_object_copies":16830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5610,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":503,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002437+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:52:23.002437+0000","last_peered":"2024-09-13T14:52:23.002437+0000","last_clean":"2024-09-13T14:52:23.002437+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:52:23.002437+0000","last_undegraded":"2024-09-13T14:52:23.002437+0000","last_fullsized":"2024-09-13T14:52:23.002437+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002848+0000","last_change":"2024-09-13T14:52:23.002552+0000","last_active":"2024-09-13T14:52:23.002848+0000","last_peered":"2024-09-13T14:52:23.002848+0000","last_clean":"2024-09-13T14:52:23.002848+0000","last_became_active":"2024-09-13T14:52:23.002224+0000","last_became_peered":"2024-09-13T14:52:23.002224+0000","last_unstale":"2024-09-13T14:52:23.002848+0000","last_undegraded":"2024-09-13T14:52:23.002848+0000","last_fullsized":"2024-09-13T14:52:23.002848+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:50:58.054308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.991363+0000","last_change":"2024-09-13T14:40:02.370166+0000","last_active":"2024-09-13T14:52:22.991363+0000","last_peered":"2024-09-13T14:52:22.991363+0000","last_clean":"2024-09-13T14:52:22.991363+0000","last_became_active":"2024-09-13T14:40:02.369614+0000","last_became_peered":"2024-09-13T14:40:02.369614+0000","last_unstale":"2024-09-13T14:52:22.991363+0000","last_undegraded":"2024-09-13T14:52:22.991363+0000","last_fullsized":"2024-09-13T14:52:22.991363+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:46.314254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.1","version":"265'5640","reported_seq":5678,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.394878+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:52:31.394878+0000","last_peered":"2024-09-13T14:52:31.394878+0000","last_clean":"2024-09-13T14:52:31.394878+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:52:31.394878+0000","last_undegraded":"2024-09-13T14:52:31.394878+0000","last_fullsized":"2024-09-13T14:52:31.394878+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5640,"log_dups_size":0,"ondisk_log_size":5640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369623040,"num_objects":5640,"num_object_clones":0,"num_object_copies":16920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5640,"num_write_kb":360960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990615+0000","last_change":"2024-09-13T14:40:02.367717+0000","last_active":"2024-09-13T14:52:22.990615+0000","last_peered":"2024-09-13T14:52:22.990615+0000","last_clean":"2024-09-13T14:52:22.990615+0000","last_became_active":"2024-09-13T14:40:02.367272+0000","last_became_peered":"2024-09-13T14:40:02.367272+0000","last_unstale":"2024-09-13T14:52:22.990615+0000","last_undegraded":"2024-09-13T14:52:22.990615+0000","last_fullsized":"2024-09-13T14:52:22.990615+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:33:42.894102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":243,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992134+0000","last_change":"2024-09-13T14:40:02.368905+0000","last_active":"2024-09-13T14:52:22.992134+0000","last_peered":"2024-09-13T14:52:22.992134+0000","last_clean":"2024-09-13T14:52:22.992134+0000","last_became_active":"2024-09-13T14:40:02.368540+0000","last_became_peered":"2024-09-13T14:40:02.368540+0000","last_unstale":"2024-09-13T14:52:22.992134+0000","last_undegraded":"2024-09-13T14:52:22.992134+0000","last_fullsized":"2024-09-13T14:52:22.992134+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:24.549430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.0","version":"265'5516","reported_seq":5570,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:28.331407+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:52:28.331407+0000","last_peered":"2024-09-13T14:52:28.331407+0000","last_clean":"2024-09-13T14:52:28.331407+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:52:28.331407+0000","last_undegraded":"2024-09-13T14:52:28.331407+0000","last_fullsized":"2024-09-13T14:52:28.331407+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:30.461258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5516,"num_write_kb":353024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":503,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002971+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:52:23.002971+0000","last_peered":"2024-09-13T14:52:23.002971+0000","last_clean":"2024-09-13T14:52:23.002971+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:52:23.002971+0000","last_undegraded":"2024-09-13T14:52:23.002971+0000","last_fullsized":"2024-09-13T14:52:23.002971+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":354,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016647+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:52:25.016647+0000","last_peered":"2024-09-13T14:52:25.016647+0000","last_clean":"2024-09-13T14:52:25.016647+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:52:25.016647+0000","last_undegraded":"2024-09-13T14:52:25.016647+0000","last_fullsized":"2024-09-13T14:52:25.016647+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":353,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.006311+0000","last_change":"2024-09-13T14:40:37.185763+0000","last_active":"2024-09-13T14:52:25.006311+0000","last_peered":"2024-09-13T14:52:25.006311+0000","last_clean":"2024-09-13T14:52:25.006311+0000","last_became_active":"2024-09-13T14:40:37.185422+0000","last_became_peered":"2024-09-13T14:40:37.185422+0000","last_unstale":"2024-09-13T14:52:25.006311+0000","last_undegraded":"2024-09-13T14:52:25.006311+0000","last_fullsized":"2024-09-13T14:52:25.006311+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:23:31.161587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"265'5738","reported_seq":5792,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.027880+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:52:31.027880+0000","last_peered":"2024-09-13T14:52:31.027880+0000","last_clean":"2024-09-13T14:52:31.027880+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:52:31.027880+0000","last_undegraded":"2024-09-13T14:52:31.027880+0000","last_fullsized":"2024-09-13T14:52:31.027880+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:05:11.218442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376045568,"num_objects":5738,"num_object_clones":0,"num_object_copies":17214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5738,"num_write_kb":367232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":491,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992805+0000","last_change":"2024-09-13T14:51:40.799562+0000","last_active":"2024-09-13T14:52:22.992805+0000","last_peered":"2024-09-13T14:52:22.992805+0000","last_clean":"2024-09-13T14:52:22.992805+0000","last_became_active":"2024-09-13T14:51:40.799144+0000","last_became_peered":"2024-09-13T14:51:40.799144+0000","last_unstale":"2024-09-13T14:52:22.992805+0000","last_undegraded":"2024-09-13T14:52:22.992805+0000","last_fullsized":"2024-09-13T14:52:22.992805+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:17:01.166260+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002500+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:52:23.002500+0000","last_peered":"2024-09-13T14:52:23.002500+0000","last_clean":"2024-09-13T14:52:23.002500+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:52:23.002500+0000","last_undegraded":"2024-09-13T14:52:23.002500+0000","last_fullsized":"2024-09-13T14:52:23.002500+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":339,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.018525+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:52:25.018525+0000","last_peered":"2024-09-13T14:52:25.018525+0000","last_clean":"2024-09-13T14:52:25.018525+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:52:25.018525+0000","last_undegraded":"2024-09-13T14:52:25.018525+0000","last_fullsized":"2024-09-13T14:52:25.018525+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":220,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.993581+0000","last_change":"2024-09-13T14:20:46.006886+0000","last_active":"2024-09-13T14:52:22.993581+0000","last_peered":"2024-09-13T14:52:22.993581+0000","last_clean":"2024-09-13T14:52:22.993581+0000","last_became_active":"2024-09-13T14:20:46.006593+0000","last_became_peered":"2024-09-13T14:20:46.006593+0000","last_unstale":"2024-09-13T14:52:22.993581+0000","last_undegraded":"2024-09-13T14:52:22.993581+0000","last_fullsized":"2024-09-13T14:52:22.993581+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:52:02.442747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"265'5702","reported_seq":8133,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.032259+0000","last_change":"2024-09-13T14:51:54.914091+0000","last_active":"2024-09-13T14:52:31.032259+0000","last_peered":"2024-09-13T14:52:31.032259+0000","last_clean":"2024-09-13T14:52:31.032259+0000","last_became_active":"2024-09-13T14:51:54.875147+0000","last_became_peered":"2024-09-13T14:51:54.875147+0000","last_unstale":"2024-09-13T14:52:31.032259+0000","last_undegraded":"2024-09-13T14:52:31.032259+0000","last_fullsized":"2024-09-13T14:52:31.032259+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:42.245089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373686272,"num_objects":5702,"num_object_clones":0,"num_object_copies":17106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5702,"num_write_kb":364928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":613,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005777+0000","last_change":"2024-09-13T14:42:56.684109+0000","last_active":"2024-09-13T14:52:25.005777+0000","last_peered":"2024-09-13T14:52:25.005777+0000","last_clean":"2024-09-13T14:52:25.005777+0000","last_became_active":"2024-09-13T14:42:56.283187+0000","last_became_peered":"2024-09-13T14:42:56.283187+0000","last_unstale":"2024-09-13T14:52:25.005777+0000","last_undegraded":"2024-09-13T14:52:25.005777+0000","last_fullsized":"2024-09-13T14:52:25.005777+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:16.528046+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":340,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.004961+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:52:25.004961+0000","last_peered":"2024-09-13T14:52:25.004961+0000","last_clean":"2024-09-13T14:52:25.004961+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:52:25.004961+0000","last_undegraded":"2024-09-13T14:52:25.004961+0000","last_fullsized":"2024-09-13T14:52:25.004961+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.008999+0000","last_change":"2024-09-13T14:51:20.405045+0000","last_active":"2024-09-13T14:52:25.008999+0000","last_peered":"2024-09-13T14:52:25.008999+0000","last_clean":"2024-09-13T14:52:25.008999+0000","last_became_active":"2024-09-13T14:51:20.404726+0000","last_became_peered":"2024-09-13T14:51:20.404726+0000","last_unstale":"2024-09-13T14:52:25.008999+0000","last_undegraded":"2024-09-13T14:52:25.008999+0000","last_fullsized":"2024-09-13T14:52:25.008999+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:12:04.185004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":286,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005270+0000","last_change":"2024-09-13T14:51:37.792733+0000","last_active":"2024-09-13T14:52:25.005270+0000","last_peered":"2024-09-13T14:52:25.005270+0000","last_clean":"2024-09-13T14:52:25.005270+0000","last_became_active":"2024-09-13T14:51:37.791658+0000","last_became_peered":"2024-09-13T14:51:37.791658+0000","last_unstale":"2024-09-13T14:52:25.005270+0000","last_undegraded":"2024-09-13T14:52:25.005270+0000","last_fullsized":"2024-09-13T14:52:25.005270+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:10.654606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.8","version":"265'5647","reported_seq":5701,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.045687+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:52:31.045687+0000","last_peered":"2024-09-13T14:52:31.045687+0000","last_clean":"2024-09-13T14:52:31.045687+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:52:31.045687+0000","last_undegraded":"2024-09-13T14:52:31.045687+0000","last_fullsized":"2024-09-13T14:52:31.045687+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:17.086610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370081792,"num_objects":5647,"num_object_clones":0,"num_object_copies":16941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":361408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":531,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.010429+0000","last_change":"2024-09-13T14:51:57.042073+0000","last_active":"2024-09-13T14:52:25.010429+0000","last_peered":"2024-09-13T14:52:25.010429+0000","last_clean":"2024-09-13T14:52:25.010429+0000","last_became_active":"2024-09-13T14:51:23.788033+0000","last_became_peered":"2024-09-13T14:51:23.788033+0000","last_unstale":"2024-09-13T14:52:25.010429+0000","last_undegraded":"2024-09-13T14:52:25.010429+0000","last_fullsized":"2024-09-13T14:52:25.010429+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:44:43.977371+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":392,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012238+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:52:25.012238+0000","last_peered":"2024-09-13T14:52:25.012238+0000","last_clean":"2024-09-13T14:52:25.012238+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:52:25.012238+0000","last_undegraded":"2024-09-13T14:52:25.012238+0000","last_fullsized":"2024-09-13T14:52:25.012238+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":352,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005367+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:52:25.005367+0000","last_peered":"2024-09-13T14:52:25.005367+0000","last_clean":"2024-09-13T14:52:25.005367+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:52:25.005367+0000","last_undegraded":"2024-09-13T14:52:25.005367+0000","last_fullsized":"2024-09-13T14:52:25.005367+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":402,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.006182+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:52:25.006182+0000","last_peered":"2024-09-13T14:52:25.006182+0000","last_clean":"2024-09-13T14:52:25.006182+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:52:25.006182+0000","last_undegraded":"2024-09-13T14:52:25.006182+0000","last_fullsized":"2024-09-13T14:52:25.006182+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"265'5693","reported_seq":5754,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.040309+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:52:31.040309+0000","last_peered":"2024-09-13T14:52:31.040309+0000","last_clean":"2024-09-13T14:52:31.040309+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:52:31.040309+0000","last_undegraded":"2024-09-13T14:52:31.040309+0000","last_fullsized":"2024-09-13T14:52:31.040309+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:35.454910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373096448,"num_objects":5693,"num_object_clones":0,"num_object_copies":17079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":537,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002301+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:52:23.002301+0000","last_peered":"2024-09-13T14:52:23.002301+0000","last_clean":"2024-09-13T14:52:23.002301+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:52:23.002301+0000","last_undegraded":"2024-09-13T14:52:23.002301+0000","last_fullsized":"2024-09-13T14:52:23.002301+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":249,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003251+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:52:23.003251+0000","last_peered":"2024-09-13T14:52:23.003251+0000","last_clean":"2024-09-13T14:52:23.003251+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:52:23.003251+0000","last_undegraded":"2024-09-13T14:52:23.003251+0000","last_fullsized":"2024-09-13T14:52:23.003251+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":290,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016173+0000","last_change":"2024-09-13T14:51:20.411788+0000","last_active":"2024-09-13T14:52:25.016173+0000","last_peered":"2024-09-13T14:52:25.016173+0000","last_clean":"2024-09-13T14:52:25.016173+0000","last_became_active":"2024-09-13T14:51:20.411434+0000","last_became_peered":"2024-09-13T14:51:20.411434+0000","last_unstale":"2024-09-13T14:52:25.016173+0000","last_undegraded":"2024-09-13T14:52:25.016173+0000","last_fullsized":"2024-09-13T14:52:25.016173+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:23:14.125924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":364,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005868+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:52:25.005868+0000","last_peered":"2024-09-13T14:52:25.005868+0000","last_clean":"2024-09-13T14:52:25.005868+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:52:25.005868+0000","last_undegraded":"2024-09-13T14:52:25.005868+0000","last_fullsized":"2024-09-13T14:52:25.005868+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"265'5699","reported_seq":5753,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.122358+0000","last_change":"2024-09-13T14:51:16.448773+0000","last_active":"2024-09-13T14:52:31.122358+0000","last_peered":"2024-09-13T14:52:31.122358+0000","last_clean":"2024-09-13T14:52:31.122358+0000","last_became_active":"2024-09-13T14:51:16.448426+0000","last_became_peered":"2024-09-13T14:51:16.448426+0000","last_unstale":"2024-09-13T14:52:31.122358+0000","last_undegraded":"2024-09-13T14:52:31.122358+0000","last_fullsized":"2024-09-13T14:52:31.122358+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:09.361383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373489664,"num_objects":5699,"num_object_clones":0,"num_object_copies":17097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5699,"num_write_kb":364736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":511,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992558+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:52:22.992558+0000","last_peered":"2024-09-13T14:52:22.992558+0000","last_clean":"2024-09-13T14:52:22.992558+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:52:22.992558+0000","last_undegraded":"2024-09-13T14:52:22.992558+0000","last_fullsized":"2024-09-13T14:52:22.992558+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":250,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003341+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:52:23.003341+0000","last_peered":"2024-09-13T14:52:23.003341+0000","last_clean":"2024-09-13T14:52:23.003341+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:52:23.003341+0000","last_undegraded":"2024-09-13T14:52:23.003341+0000","last_fullsized":"2024-09-13T14:52:23.003341+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":416,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.009833+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:52:25.009833+0000","last_peered":"2024-09-13T14:52:25.009833+0000","last_clean":"2024-09-13T14:52:25.009833+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:52:25.009833+0000","last_undegraded":"2024-09-13T14:52:25.009833+0000","last_fullsized":"2024-09-13T14:52:25.009833+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":273,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005265+0000","last_change":"2024-09-13T14:41:21.967042+0000","last_active":"2024-09-13T14:52:25.005265+0000","last_peered":"2024-09-13T14:52:25.005265+0000","last_clean":"2024-09-13T14:52:25.005265+0000","last_became_active":"2024-09-13T14:41:21.966728+0000","last_became_peered":"2024-09-13T14:41:21.966728+0000","last_unstale":"2024-09-13T14:52:25.005265+0000","last_undegraded":"2024-09-13T14:52:25.005265+0000","last_fullsized":"2024-09-13T14:52:25.005265+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:15.023636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"265'5488","reported_seq":5542,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:28.381741+0000","last_change":"2024-09-13T14:51:16.448079+0000","last_active":"2024-09-13T14:52:28.381741+0000","last_peered":"2024-09-13T14:52:28.381741+0000","last_clean":"2024-09-13T14:52:28.381741+0000","last_became_active":"2024-09-13T14:51:16.447056+0000","last_became_peered":"2024-09-13T14:51:16.447056+0000","last_unstale":"2024-09-13T14:52:28.381741+0000","last_undegraded":"2024-09-13T14:52:28.381741+0000","last_fullsized":"2024-09-13T14:52:28.381741+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:07:28.127215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359661568,"num_objects":5488,"num_object_clones":0,"num_object_copies":16464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5488,"num_write_kb":351232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":545,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992238+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:52:22.992238+0000","last_peered":"2024-09-13T14:52:22.992238+0000","last_clean":"2024-09-13T14:52:22.992238+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:52:22.992238+0000","last_undegraded":"2024-09-13T14:52:22.992238+0000","last_fullsized":"2024-09-13T14:52:22.992238+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":351,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.009736+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:52:25.009736+0000","last_peered":"2024-09-13T14:52:25.009736+0000","last_clean":"2024-09-13T14:52:25.009736+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:52:25.009736+0000","last_undegraded":"2024-09-13T14:52:25.009736+0000","last_fullsized":"2024-09-13T14:52:25.009736+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.008759+0000","last_change":"2024-09-13T14:51:20.403571+0000","last_active":"2024-09-13T14:52:25.008759+0000","last_peered":"2024-09-13T14:52:25.008759+0000","last_clean":"2024-09-13T14:52:25.008759+0000","last_became_active":"2024-09-13T14:51:20.403244+0000","last_became_peered":"2024-09-13T14:51:20.403244+0000","last_unstale":"2024-09-13T14:52:25.008759+0000","last_undegraded":"2024-09-13T14:52:25.008759+0000","last_fullsized":"2024-09-13T14:52:25.008759+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:08.930829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.013267+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:52:25.013267+0000","last_peered":"2024-09-13T14:52:25.013267+0000","last_clean":"2024-09-13T14:52:25.013267+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:52:25.013267+0000","last_undegraded":"2024-09-13T14:52:25.013267+0000","last_fullsized":"2024-09-13T14:52:25.013267+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"265'5565","reported_seq":5619,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:28.398292+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:52:28.398292+0000","last_peered":"2024-09-13T14:52:28.398292+0000","last_clean":"2024-09-13T14:52:28.398292+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:52:28.398292+0000","last_undegraded":"2024-09-13T14:52:28.398292+0000","last_fullsized":"2024-09-13T14:52:28.398292+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:59.672326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364707840,"num_objects":5565,"num_object_clones":0,"num_object_copies":16695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5565,"num_write_kb":356160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":503,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.991508+0000","last_change":"2024-09-13T14:43:00.868011+0000","last_active":"2024-09-13T14:52:22.991508+0000","last_peered":"2024-09-13T14:52:22.991508+0000","last_clean":"2024-09-13T14:52:22.991508+0000","last_became_active":"2024-09-13T14:42:56.283775+0000","last_became_peered":"2024-09-13T14:42:56.283775+0000","last_unstale":"2024-09-13T14:52:22.991508+0000","last_undegraded":"2024-09-13T14:52:22.991508+0000","last_fullsized":"2024-09-13T14:52:22.991508+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:43:00.867870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:43:00.867870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:55:25.806175+0000","scrub_duration":0.0043830680000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":362,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.013765+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:52:25.013765+0000","last_peered":"2024-09-13T14:52:25.013765+0000","last_clean":"2024-09-13T14:52:25.013765+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:52:25.013765+0000","last_undegraded":"2024-09-13T14:52:25.013765+0000","last_fullsized":"2024-09-13T14:52:25.013765+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":352,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005673+0000","last_change":"2024-09-13T14:35:58.848332+0000","last_active":"2024-09-13T14:52:25.005673+0000","last_peered":"2024-09-13T14:52:25.005673+0000","last_clean":"2024-09-13T14:52:25.005673+0000","last_became_active":"2024-09-13T14:35:58.847943+0000","last_became_peered":"2024-09-13T14:35:58.847943+0000","last_unstale":"2024-09-13T14:52:25.005673+0000","last_undegraded":"2024-09-13T14:52:25.005673+0000","last_fullsized":"2024-09-13T14:52:25.005673+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:38:39.152579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":358,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016405+0000","last_change":"2024-09-13T14:51:37.793493+0000","last_active":"2024-09-13T14:52:25.016405+0000","last_peered":"2024-09-13T14:52:25.016405+0000","last_clean":"2024-09-13T14:52:25.016405+0000","last_became_active":"2024-09-13T14:51:37.792802+0000","last_became_peered":"2024-09-13T14:51:37.792802+0000","last_unstale":"2024-09-13T14:52:25.016405+0000","last_undegraded":"2024-09-13T14:52:25.016405+0000","last_fullsized":"2024-09-13T14:52:25.016405+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:44:51.004042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"265'5802","reported_seq":5832,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:31.425354+0000","last_change":"2024-09-13T14:51:17.034483+0000","last_active":"2024-09-13T14:52:31.425354+0000","last_peered":"2024-09-13T14:52:31.425354+0000","last_clean":"2024-09-13T14:52:31.425354+0000","last_became_active":"2024-09-13T14:51:17.034153+0000","last_became_peered":"2024-09-13T14:51:17.034153+0000","last_unstale":"2024-09-13T14:52:31.425354+0000","last_undegraded":"2024-09-13T14:52:31.425354+0000","last_fullsized":"2024-09-13T14:52:31.425354+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":5802,"log_dups_size":0,"ondisk_log_size":5802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:40:25.506330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380239872,"num_objects":5802,"num_object_clones":0,"num_object_copies":17406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":371328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":612,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012671+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:52:25.012671+0000","last_peered":"2024-09-13T14:52:25.012671+0000","last_clean":"2024-09-13T14:52:25.012671+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:52:25.012671+0000","last_undegraded":"2024-09-13T14:52:25.012671+0000","last_fullsized":"2024-09-13T14:52:25.012671+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":213,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005425+0000","last_change":"2024-09-13T14:51:37.797640+0000","last_active":"2024-09-13T14:52:25.005425+0000","last_peered":"2024-09-13T14:52:25.005425+0000","last_clean":"2024-09-13T14:52:25.005425+0000","last_became_active":"2024-09-13T14:51:37.791624+0000","last_became_peered":"2024-09-13T14:51:37.791624+0000","last_unstale":"2024-09-13T14:52:25.005425+0000","last_undegraded":"2024-09-13T14:52:25.005425+0000","last_fullsized":"2024-09-13T14:52:25.005425+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:34.248753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":248,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992707+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:52:22.992707+0000","last_peered":"2024-09-13T14:52:22.992707+0000","last_clean":"2024-09-13T14:52:22.992707+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:52:22.992707+0000","last_undegraded":"2024-09-13T14:52:22.992707+0000","last_fullsized":"2024-09-13T14:52:22.992707+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":220,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990110+0000","last_change":"2024-09-13T14:20:46.806316+0000","last_active":"2024-09-13T14:52:22.990110+0000","last_peered":"2024-09-13T14:52:22.990110+0000","last_clean":"2024-09-13T14:52:22.990110+0000","last_became_active":"2024-09-13T14:20:46.806036+0000","last_became_peered":"2024-09-13T14:20:46.806036+0000","last_unstale":"2024-09-13T14:52:22.990110+0000","last_undegraded":"2024-09-13T14:52:22.990110+0000","last_fullsized":"2024-09-13T14:52:22.990110+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:36:10.302967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990858+0000","last_change":"2024-09-13T14:35:58.846596+0000","last_active":"2024-09-13T14:52:22.990858+0000","last_peered":"2024-09-13T14:52:22.990858+0000","last_clean":"2024-09-13T14:52:22.990858+0000","last_became_active":"2024-09-13T14:35:58.846169+0000","last_became_peered":"2024-09-13T14:35:58.846169+0000","last_unstale":"2024-09-13T14:52:22.990858+0000","last_undegraded":"2024-09-13T14:52:22.990858+0000","last_fullsized":"2024-09-13T14:52:22.990858+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:05:23.621167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":242,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990197+0000","last_change":"2024-09-13T14:20:45.776446+0000","last_active":"2024-09-13T14:52:22.990197+0000","last_peered":"2024-09-13T14:52:22.990197+0000","last_clean":"2024-09-13T14:52:22.990197+0000","last_became_active":"2024-09-13T14:20:45.776164+0000","last_became_peered":"2024-09-13T14:20:45.776164+0000","last_unstale":"2024-09-13T14:52:22.990197+0000","last_undegraded":"2024-09-13T14:52:22.990197+0000","last_fullsized":"2024-09-13T14:52:22.990197+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:08.209451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":272,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005484+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:52:25.005484+0000","last_peered":"2024-09-13T14:52:25.005484+0000","last_clean":"2024-09-13T14:52:25.005484+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:52:25.005484+0000","last_undegraded":"2024-09-13T14:52:25.005484+0000","last_fullsized":"2024-09-13T14:52:25.005484+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":298,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012414+0000","last_change":"2024-09-13T14:40:38.172727+0000","last_active":"2024-09-13T14:52:25.012414+0000","last_peered":"2024-09-13T14:52:25.012414+0000","last_clean":"2024-09-13T14:52:25.012414+0000","last_became_active":"2024-09-13T14:40:38.172309+0000","last_became_peered":"2024-09-13T14:40:38.172309+0000","last_unstale":"2024-09-13T14:52:25.012414+0000","last_undegraded":"2024-09-13T14:52:25.012414+0000","last_fullsized":"2024-09-13T14:52:25.012414+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:36:37.892208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":233,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002953+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:52:23.002953+0000","last_peered":"2024-09-13T14:52:23.002953+0000","last_clean":"2024-09-13T14:52:23.002953+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:52:23.002953+0000","last_undegraded":"2024-09-13T14:52:23.002953+0000","last_fullsized":"2024-09-13T14:52:23.002953+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":339,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012587+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:52:25.012587+0000","last_peered":"2024-09-13T14:52:25.012587+0000","last_clean":"2024-09-13T14:52:25.012587+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:52:25.012587+0000","last_undegraded":"2024-09-13T14:52:25.012587+0000","last_fullsized":"2024-09-13T14:52:25.012587+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.008845+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:52:25.008845+0000","last_peered":"2024-09-13T14:52:25.008845+0000","last_clean":"2024-09-13T14:52:25.008845+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:52:25.008845+0000","last_undegraded":"2024-09-13T14:52:25.008845+0000","last_fullsized":"2024-09-13T14:52:25.008845+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":352,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016311+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:52:25.016311+0000","last_peered":"2024-09-13T14:52:25.016311+0000","last_clean":"2024-09-13T14:52:25.016311+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:52:25.016311+0000","last_undegraded":"2024-09-13T14:52:25.016311+0000","last_fullsized":"2024-09-13T14:52:25.016311+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.992632+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:52:22.992632+0000","last_peered":"2024-09-13T14:52:22.992632+0000","last_clean":"2024-09-13T14:52:22.992632+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:52:22.992632+0000","last_undegraded":"2024-09-13T14:52:22.992632+0000","last_fullsized":"2024-09-13T14:52:22.992632+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":388,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.016492+0000","last_change":"2024-09-13T14:51:37.794313+0000","last_active":"2024-09-13T14:52:25.016492+0000","last_peered":"2024-09-13T14:52:25.016492+0000","last_clean":"2024-09-13T14:52:25.016492+0000","last_became_active":"2024-09-13T14:51:37.792700+0000","last_became_peered":"2024-09-13T14:51:37.792700+0000","last_unstale":"2024-09-13T14:52:25.016492+0000","last_undegraded":"2024-09-13T14:52:25.016492+0000","last_fullsized":"2024-09-13T14:52:25.016492+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:36:17.804238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":44,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.010454+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:52:25.010454+0000","last_peered":"2024-09-13T14:52:25.010454+0000","last_clean":"2024-09-13T14:52:25.010454+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:52:25.010454+0000","last_undegraded":"2024-09-13T14:52:25.010454+0000","last_fullsized":"2024-09-13T14:52:25.010454+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":350,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.005144+0000","last_change":"2024-09-13T14:40:37.184569+0000","last_active":"2024-09-13T14:52:25.005144+0000","last_peered":"2024-09-13T14:52:25.005144+0000","last_clean":"2024-09-13T14:52:25.005144+0000","last_became_active":"2024-09-13T14:40:37.184217+0000","last_became_peered":"2024-09-13T14:40:37.184217+0000","last_unstale":"2024-09-13T14:52:25.005144+0000","last_undegraded":"2024-09-13T14:52:25.005144+0000","last_fullsized":"2024-09-13T14:52:25.005144+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:20.674728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":351,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.013166+0000","last_change":"2024-09-13T14:40:37.182591+0000","last_active":"2024-09-13T14:52:25.013166+0000","last_peered":"2024-09-13T14:52:25.013166+0000","last_clean":"2024-09-13T14:52:25.013166+0000","last_became_active":"2024-09-13T14:40:37.182153+0000","last_became_peered":"2024-09-13T14:40:37.182153+0000","last_unstale":"2024-09-13T14:52:25.013166+0000","last_undegraded":"2024-09-13T14:52:25.013166+0000","last_fullsized":"2024-09-13T14:52:25.013166+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:10.855412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":275,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.015084+0000","last_change":"2024-09-13T14:40:38.171380+0000","last_active":"2024-09-13T14:52:25.015084+0000","last_peered":"2024-09-13T14:52:25.015084+0000","last_clean":"2024-09-13T14:52:25.015084+0000","last_became_active":"2024-09-13T14:40:38.170222+0000","last_became_peered":"2024-09-13T14:40:38.170222+0000","last_unstale":"2024-09-13T14:52:25.015084+0000","last_undegraded":"2024-09-13T14:52:25.015084+0000","last_fullsized":"2024-09-13T14:52:25.015084+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:45:05.086265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":378,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012084+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:52:25.012084+0000","last_peered":"2024-09-13T14:52:25.012084+0000","last_clean":"2024-09-13T14:52:25.012084+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:52:25.012084+0000","last_undegraded":"2024-09-13T14:52:25.012084+0000","last_fullsized":"2024-09-13T14:52:25.012084+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":294,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012698+0000","last_change":"2024-09-13T14:51:37.794439+0000","last_active":"2024-09-13T14:52:25.012698+0000","last_peered":"2024-09-13T14:52:25.012698+0000","last_clean":"2024-09-13T14:52:25.012698+0000","last_became_active":"2024-09-13T14:51:37.793627+0000","last_became_peered":"2024-09-13T14:51:37.793627+0000","last_unstale":"2024-09-13T14:52:25.012698+0000","last_undegraded":"2024-09-13T14:52:25.012698+0000","last_fullsized":"2024-09-13T14:52:25.012698+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:30.400602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":247,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990797+0000","last_change":"2024-09-13T14:40:37.187316+0000","last_active":"2024-09-13T14:52:22.990797+0000","last_peered":"2024-09-13T14:52:22.990797+0000","last_clean":"2024-09-13T14:52:22.990797+0000","last_became_active":"2024-09-13T14:40:37.186990+0000","last_became_peered":"2024-09-13T14:40:37.186990+0000","last_unstale":"2024-09-13T14:52:22.990797+0000","last_undegraded":"2024-09-13T14:52:22.990797+0000","last_fullsized":"2024-09-13T14:52:22.990797+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:51:25.164342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.002706+0000","last_change":"2024-09-13T14:35:58.859244+0000","last_active":"2024-09-13T14:52:23.002706+0000","last_peered":"2024-09-13T14:52:23.002706+0000","last_clean":"2024-09-13T14:52:23.002706+0000","last_became_active":"2024-09-13T14:35:58.857419+0000","last_became_peered":"2024-09-13T14:35:58.857419+0000","last_unstale":"2024-09-13T14:52:23.002706+0000","last_undegraded":"2024-09-13T14:52:23.002706+0000","last_fullsized":"2024-09-13T14:52:23.002706+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:06.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":265,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.990321+0000","last_change":"2024-09-13T14:51:20.399203+0000","last_active":"2024-09-13T14:52:22.990321+0000","last_peered":"2024-09-13T14:52:22.990321+0000","last_clean":"2024-09-13T14:52:22.990321+0000","last_became_active":"2024-09-13T14:51:20.398824+0000","last_became_peered":"2024-09-13T14:51:20.398824+0000","last_unstale":"2024-09-13T14:52:22.990321+0000","last_undegraded":"2024-09-13T14:52:22.990321+0000","last_fullsized":"2024-09-13T14:52:22.990321+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:35.733324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":219,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:23.003238+0000","last_change":"2024-09-13T14:20:45.773426+0000","last_active":"2024-09-13T14:52:23.003238+0000","last_peered":"2024-09-13T14:52:23.003238+0000","last_clean":"2024-09-13T14:52:23.003238+0000","last_became_active":"2024-09-13T14:20:45.773142+0000","last_became_peered":"2024-09-13T14:20:45.773142+0000","last_unstale":"2024-09-13T14:52:23.003238+0000","last_undegraded":"2024-09-13T14:52:23.003238+0000","last_fullsized":"2024-09-13T14:52:23.003238+0000","mapping_epoch":67,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:08.555905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":397,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012943+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:52:25.012943+0000","last_peered":"2024-09-13T14:52:25.012943+0000","last_clean":"2024-09-13T14:52:25.012943+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:52:25.012943+0000","last_undegraded":"2024-09-13T14:52:25.012943+0000","last_fullsized":"2024-09-13T14:52:25.012943+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":373,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.013601+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:52:25.013601+0000","last_peered":"2024-09-13T14:52:25.013601+0000","last_clean":"2024-09-13T14:52:25.013601+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:52:25.013601+0000","last_undegraded":"2024-09-13T14:52:25.013601+0000","last_fullsized":"2024-09-13T14:52:25.013601+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":143,"reported_epoch":264,"state":"active+clean","last_fresh":"2024-09-13T14:52:22.989890+0000","last_change":"2024-09-13T14:41:21.967314+0000","last_active":"2024-09-13T14:52:22.989890+0000","last_peered":"2024-09-13T14:52:22.989890+0000","last_clean":"2024-09-13T14:52:22.989890+0000","last_became_active":"2024-09-13T14:41:21.966848+0000","last_became_peered":"2024-09-13T14:41:21.966848+0000","last_unstale":"2024-09-13T14:52:22.989890+0000","last_undegraded":"2024-09-13T14:52:22.989890+0000","last_fullsized":"2024-09-13T14:52:22.989890+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:17:54.191016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":137,"reported_epoch":265,"state":"active+clean","last_fresh":"2024-09-13T14:52:25.012197+0000","last_change":"2024-09-13T14:51:37.788879+0000","last_active":"2024-09-13T14:52:25.012197+0000","last_peered":"2024-09-13T14:52:25.012197+0000","last_clean":"2024-09-13T14:52:25.012197+0000","last_became_active":"2024-09-13T14:51:37.785491+0000","last_became_peered":"2024-09-13T14:51:37.785491+0000","last_unstale":"2024-09-13T14:52:25.012197+0000","last_undegraded":"2024-09-13T14:52:25.012197+0000","last_fullsized":"2024-09-13T14:52:25.012197+0000","mapping_epoch":255,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:42:33.630487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":5930942464,"num_objects":90499,"num_object_clones":0,"num_object_copies":271497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":90499,"num_write_kb":5791936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17792303104,"data_stored":17792303104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90499,"ondisk_log_size":90499,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1320,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234389,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3048940,"kb_used_data":2187620,"kb_used_omap":1,"kb_used_meta":861310,"kb_avail":90696212,"statfs":{"total":95995035648,"available":92872921088,"internally_reserved":0,"allocated":2240122880,"data_stored":2238716846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":881981899},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496027,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3762156,"kb_used_data":2918884,"kb_used_omap":1,"kb_used_meta":843262,"kb_avail":89982996,"statfs":{"total":95995035648,"available":92142587904,"internally_reserved":0,"allocated":2988937216,"data_stored":2987531727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":863500743},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757665,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2469280,"kb_used_data":1449624,"kb_used_omap":1,"kb_used_meta":1019646,"kb_avail":91275872,"statfs":{"total":95995035648,"available":93466492928,"internally_reserved":0,"allocated":1484414976,"data_stored":1483017232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1044117958},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019301,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2344680,"kb_used_data":1457120,"kb_used_omap":1,"kb_used_meta":887550,"kb_avail":91400472,"statfs":{"total":95995035648,"available":93594083328,"internally_reserved":0,"allocated":1492090880,"data_stored":1490688928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":908851654},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650513,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2975964,"kb_used_data":2521108,"kb_used_omap":4,"kb_used_meta":454843,"kb_avail":90769188,"statfs":{"total":95995035648,"available":92947648512,"internally_reserved":0,"allocated":2581614592,"data_stored":2580220706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":465759791},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879448,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2577316,"kb_used_data":2182748,"kb_used_omap":6,"kb_used_meta":394553,"kb_avail":91167836,"statfs":{"total":95995035648,"available":93355864064,"internally_reserved":0,"allocated":2235133952,"data_stored":2233732490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":404022841},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435464,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3859864,"kb_used_data":3168272,"kb_used_omap":5,"kb_used_meta":691578,"kb_avail":89885288,"statfs":{"total":95995035648,"available":92042534912,"internally_reserved":0,"allocated":3244310528,"data_stored":3242919633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":708176225},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926039,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751508,"kb_used_data":1404812,"kb_used_omap":5,"kb_used_meta":346682,"kb_avail":91993644,"statfs":{"total":95995035648,"available":94201491456,"internally_reserved":0,"allocated":1438527488,"data_stored":1437141160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":355002538},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1450573824,"data_stored":1450573824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3266576384,"data_stored":3266576384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2247688192,"data_stored":2247688192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2600468480,"data_stored":2600468480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1498284032,"data_stored":1498284032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1487601664,"data_stored":1487601664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2988769280,"data_stored":2988769280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2252341248,"data_stored":2252341248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:52:35.863 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_6 pgp_num 16 2024-09-13T14:52:35.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.865 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:35.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:35.866 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:35.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:35.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:35.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:36.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:36.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:36.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:36.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:36.303 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:36.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[108570]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[108570]: pgmap v910: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[113926]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[113926]: pgmap v910: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:52:36.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:36.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:36.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:36.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:36.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:36 smithi159 ceph-mon[102519]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:52:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:36 smithi159 ceph-mon[102519]: pgmap v910: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-13T14:52:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:52:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:36.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:36.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:36.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:36.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:36.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:36.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:36.774 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 96661 96645 91.4975 63.4375 0.015957 0.0109233 2024-09-13T14:52:36.778 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:36.820 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:36.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:36.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:36.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:52:36.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:36.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:36.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:36.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:37.089 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.130 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:37.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:37.231 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:37.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:37.317 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:37.317 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:37.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:37.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:37.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:37.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:37.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:37.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:37.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:37.754 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:37.774 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 97482 97466 90.8977 51.3125 0.0042278 0.0109642 2024-09-13T14:52:37.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:37.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:37.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:37.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:37.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:38.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.047 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:38.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:38.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:38.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:38.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:38.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:38.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:38.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:38.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:38.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:38 smithi159 ceph-mon[102519]: pgmap v911: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:52:38.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:38.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:38.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:38.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:38 smithi042 ceph-mon[108570]: pgmap v911: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:52:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:38 smithi042 ceph-mon[113926]: pgmap v911: 99 pgs: 99 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:52:38.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:52:38.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:38.760 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:38.774 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 98402 98386 90.4066 57.5 0.0177713 0.0110558 2024-09-13T14:52:38.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:38.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:38.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:38.981 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:39.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:39.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:39.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:39.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:39.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:39.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:39.229 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:52:39.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:39.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:39.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:39.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:39.441 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4083415328' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.441 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:39 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:39.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:39.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:39.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4083415328' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:39 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4083415328' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:39 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:52:39.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:39.727 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:39.774 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 99476 99460 90.0691 67.125 0.0151908 0.0110973 2024-09-13T14:52:39.829 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:39.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:39.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:39.963 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:40.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:40.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:40.073 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:39] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-09-13T14:52:40.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:40.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:40.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:40.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:40.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[113926]: pgmap v912: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:52:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[113926]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[108570]: pgmap v912: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:40 smithi042 ceph-mon[108570]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:40.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.417 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:40.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:40 smithi159 ceph-mon[102519]: pgmap v912: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T14:52:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:40 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:52:40.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:40 smithi159 ceph-mon[102519]: osdmap e266: 8 total, 8 up, 8 in 2024-09-13T14:52:40.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:40.574 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:40.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:40.638 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:40.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:40.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:40.775 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 100522 100506 89.7155 65.375 0.00642693 0.0111413 2024-09-13T14:52:40.807 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:40.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:40.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:40.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:40.982 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:52:40.982 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:52:40.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:40.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:40.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:41.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:41.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:41.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:41.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:41.381 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:41.416 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:41.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:41.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:41.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:41.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:41.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:41.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:41.775 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 101498 101482 89.3111 61 0.0283989 0.0111913 2024-09-13T14:52:41.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:41.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:52:41.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:41.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:41.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:41.982 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:41.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:41.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:42.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:52:42.123 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:42.133 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:42.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:42.256 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 11 pgp_num to 16 2024-09-13T14:52:42.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:42.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:42.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:42 smithi159 ceph-mon[102519]: pgmap v914: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:52:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:42.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:42.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.596 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:42 smithi042 ceph-mon[108570]: pgmap v914: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:52:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:42 smithi042 ceph-mon[113926]: pgmap v914: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-13T14:52:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T14:52:42.697 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:42.775 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 102463 102447 88.9083 60.3125 0.0114749 0.0112421 2024-09-13T14:52:42.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:42.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:42.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:42.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:43.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:43.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:43.153 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:43.255 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:43.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:43.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.437 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.481 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:43.481 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[108570]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:43.481 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[108570]: pgmap v916: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:52:43.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:43.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[113926]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:43.482 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:43 smithi042 ceph-mon[113926]: pgmap v916: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:52:43.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:43.728 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:52:43.729 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:43.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1283289120' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_6", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T14:52:43.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:43 smithi159 ceph-mon[102519]: osdmap e267: 8 total, 8 up, 8 in 2024-09-13T14:52:43.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:43 smithi159 ceph-mon[102519]: pgmap v916: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-13T14:52:43.762 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:52:43.762 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:43.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 103395 103379 88.4861 58.25 0.0150748 0.0112956 2024-09-13T14:52:43.799 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:43.808 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:52:43.808 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:43.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:43.846 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:52:43.846 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:43.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:43.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:43.881 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:52:43.881 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:43.913 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:52:43.913 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:43.949 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:52:43.950 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:43.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:43.976 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:52:43.977 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:44.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:44.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.136 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:44.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:44.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:44.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:44.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1192617781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:44.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1192617781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:44.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:44.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:44.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1192617781' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:44.777 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 104334 104318 88.0834 58.6875 0.0280102 0.0113468 2024-09-13T14:52:44.795 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:44.826 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:44.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:44.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:44.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:44.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:45.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:45.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:45.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:45.228 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:45.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:45.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:45.368 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:45.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:45.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:45.541 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:45 smithi159 ceph-mon[102519]: pgmap v917: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-09-13T14:52:45.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:45 smithi042 ceph-mon[108570]: pgmap v917: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-09-13T14:52:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:45 smithi042 ceph-mon[113926]: pgmap v917: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-09-13T14:52:45.612 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:45.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.652 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:45.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:45.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:45.777 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 105350 105334 87.7556 63.5 0.0106427 0.0113899 2024-09-13T14:52:45.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:45.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:45.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:45.913 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:45.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:46.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:46.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:46.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:46.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:46.200 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:46.219 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:46.219 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:52:46.219 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:52:46.219 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:52:46.220 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:46.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:46.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:46.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:46.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:46.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:46.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:46.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:46.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:46.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:46.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:46.777 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 106215 106199 87.3123 54.0625 0.00751203 0.0114477 2024-09-13T14:52:46.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:46.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:46.880 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:46.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:46.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:47.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:52:47.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:47.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:47.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:47.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:47.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:47.385 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:52:47.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:47.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:47.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:47.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:47 smithi042 ceph-mon[108570]: pgmap v918: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:52:47.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:47 smithi042 ceph-mon[113926]: pgmap v918: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:52:47.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:47.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:47.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:47.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:47 smithi159 ceph-mon[102519]: pgmap v918: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-13T14:52:47.777 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 107047 107031 86.8537 52 0.0119803 0.0115087 2024-09-13T14:52:47.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:47.863 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:47.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:47.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:47.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:48.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:48.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:48.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:48.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:48.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:48.153 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:52:48.153 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:48.153 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-13T14:52:48.153 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:52:48.153 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:52:48.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-13T14:52:48.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:48.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:48.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:48.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:48.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:48.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:48.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:48.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:48.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:48.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 108016 108000 86.5149 60.5625 0.0243531 0.0115532 2024-09-13T14:52:48.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:48.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:48.961 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:48.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:48.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:49.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:49.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:49.226 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:49.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.255 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:49.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:49.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:49.396 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:49.463 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:49.497 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:49.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:49 smithi042 ceph-mon[113926]: pgmap v919: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T14:52:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:49 smithi042 ceph-mon[108570]: pgmap v919: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T14:52:49.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:49.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:49.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:49 smithi159 ceph-mon[102519]: pgmap v919: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T14:52:49.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 108990 108974 86.1903 60.875 0.0139276 0.0115969 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84558 9.3 9:c85b4db8:::benchmark_data_smithi042_157727_object84557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.795 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.15078865600003, 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.37831424299999999, 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84558 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.796 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641439+0000", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641438+0000", 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.797 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641520+0000", 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2422000000000005e-05 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641531+0000", 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0536e-05 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.798 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641537+0000", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4509999999999997e-06 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642039+0000", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050194900000000001 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642168+0000", 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012866399999999999 2024-09-13T14:52:49.799 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642290+0000", 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012242000000000001 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.647958+0000", 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056680840000000003 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.800 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719589+0000", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.071630894000000001 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019655+0000", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30006546099999998 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019711+0000", 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6632000000000003e-05 2024-09-13T14:52:49.801 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.019753+0000", 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1925999999999998e-05 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84559 9.3 9:c8ad0e78:::benchmark_data_smithi042_157727_object84558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.15064126599998, 2024-09-13T14:52:49.802 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.38015051100000002, 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84559 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.803 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641586+0000", 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.804 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.641983+0000", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039788200000000001 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642357+0000", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00037351099999999998 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.642364+0000", 2024-09-13T14:52:49.805 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2790000000000001e-06 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645661+0000", 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032972330000000001 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.806 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645784+0000", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012222900000000001 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.645882+0000", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8276999999999994e-05 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.648108+0000", 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022264009999999998 2024-09-13T14:52:49.807 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.722650+0000", 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.074541741999999994 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021646+0000", 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29899620700000001 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.808 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021712+0000", 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5488999999999996e-05 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021737+0000", 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4961e-05 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.809 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84563 9.d 9:ba52b3ee:::benchmark_data_smithi042_157727_object84562:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.072462825, 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.31030390200000002, 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.810 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84563 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.812 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719765+0000", 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719763+0000", 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719824+0000", 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0915999999999997e-05 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.813 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719836+0000", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1992e-05 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719841+0000", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.271e-06 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719885+0000", 2024-09-13T14:52:49.814 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3130000000000002e-05 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.719994+0000", 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109428 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.720091+0000", 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7001000000000002e-05 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.815 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723173+0000", 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030821899999999998 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.723401+0000", 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022781400000000001 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.029965+0000", 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30656361599999998 2024-09-13T14:52:49.816 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030017+0000", 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2584999999999999e-05 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030069+0000", 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1329999999999998e-05 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.817 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84572 9.3 9:cea63616:::benchmark_data_smithi042_157727_object84571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.06462629600003, 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.30071340699999999, 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84572 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.818 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727601+0000", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727600+0000", 2024-09-13T14:52:49.819 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727712+0000", 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112267 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727728+0000", 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6545000000000002e-05 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.820 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727736+0000", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7950000000000008e-06 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727777+0000", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0595e-05 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727921+0000", 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000143879 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.821 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728044+0000", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000123196 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730658+0000", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026146229999999999 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731148+0000", 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048908600000000001 2024-09-13T14:52:49.822 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028092+0000", 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29694452599999999 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028269+0000", 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000177126 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.823 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.028315+0000", 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5414999999999998e-05 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84573 9.e 9:7570e4c0:::benchmark_data_smithi042_157727_object84572:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.06443770099997, 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29393596500000002, 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.824 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84573 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.825 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727790+0000", 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727789+0000", 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727845+0000", 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5803000000000001e-05 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.826 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727853+0000", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3659999999999992e-06 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727857+0000", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9099999999999998e-06 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727888+0000", 2024-09-13T14:52:49.827 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0574999999999998e-05 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.727979+0000", 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1242999999999994e-05 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,4", 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.728059+0000", 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0238e-05 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.828 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730776+0000", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002716684 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.730964+0000", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018814 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021615+0000", 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29065137200000002 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.829 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021666+0000", 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0467000000000003e-05 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.021726+0000", 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9982000000000003e-05 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.830 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.34797.0:84574 9.d 9:bb2c6928:::benchmark_data_smithi042_157727_object84573:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 588.06112566700006, 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29919750000000001, 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.34797", 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3925435066", 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 84574 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.831 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731102+0000", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731101+0000", 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.832 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731161+0000", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0677999999999997e-05 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731171+0000", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7480000000000005e-06 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731177+0000", 2024-09-13T14:52:49.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3400000000000003e-06 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731204+0000", 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7103000000000001e-05 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731302+0000", 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7919999999999998e-05 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,5", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.731399+0000", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6759e-05 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733667+0000", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022682940000000001 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:01.733966+0000", 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000298528 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030141+0000", 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.29617486399999998 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030264+0000", 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012284000000000001 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:43:02.030299+0000", 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5714999999999999e-05 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96791 10.c e205/184 10:3cfbffb8:::benchmark_data_smithi042_191918_object96790:head v 205'6009, mlcod=205'6007)", 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.82293125000001, 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16168563899999999, 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.837 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969296+0000", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969295+0000", 2024-09-13T14:52:49.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969317+0000", 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1963e-05 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969329+0000", 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1556000000000001e-05 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969335+0000", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2720000000000003e-06 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969400+0000", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4843000000000002e-05 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.969419+0000", 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9174999999999999e-05 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.840 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130936+0000", 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.161517046 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130982+0000", 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5559000000000002e-05 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.841 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96796 10.c e205/184 10:3e3d83db:::benchmark_data_smithi042_191918_object96795:head v 205'6010, mlcod=205'6007)", 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.81950335599998, 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.15827959899999999, 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.842 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972724+0000", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972723+0000", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972747+0000", 2024-09-13T14:52:49.843 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3302000000000002e-05 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972757+0000", 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0682e-05 2024-09-13T14:52:49.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972764+0000", 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5060000000000001e-06 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972822+0000", 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8304999999999998e-05 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.845 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.972841+0000", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9266000000000002e-05 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130986+0000", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.158144854 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131004+0000", 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7623999999999999e-05 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.846 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96798 10.15 e205/192 10:ab2d6cee:::benchmark_data_smithi042_191918_object96797:head v 205'4774, mlcod=205'4772)", 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.818623414, 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.160394434, 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.847 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973604+0000", 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973603+0000", 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.848 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973649+0000", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5407000000000001e-05 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973660+0000", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1304999999999999e-05 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973667+0000", 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.759e-06 2024-09-13T14:52:49.849 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973725+0000", 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7856999999999998e-05 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.973744+0000", 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9891999999999999e-05 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.850 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.133917+0000", 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16017301 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.133999+0000", 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1118000000000005e-05 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96803 10.18 e205/192 10:1bbb31d9:::benchmark_data_smithi042_191918_object96802:head v 205'4706, mlcod=205'4704)", 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:52:49.851 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.81748340399997, 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.16000645999999999, 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974744+0000", 2024-09-13T14:52:49.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974743+0000", 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.974784+0000", 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0402999999999999e-05 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130893+0000", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.156109523 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130910+0000", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7244999999999998e-05 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131222+0000", 2024-09-13T14:52:49.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00031179800000000001 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131240+0000", 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7680999999999999e-05 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134683+0000", 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003443345 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134751+0000", 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7281999999999997e-05 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96804 10.19 e205/192 10:9bbfbfa6:::benchmark_data_smithi042_191918_object96803:head v 205'4741, mlcod=205'4739)", 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.81257358599998, 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.15497003200000001, 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.857 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979654+0000", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979653+0000", 2024-09-13T14:52:49.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.979677+0000", 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4644e-05 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130894+0000", 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.151217142 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.859 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.130910+0000", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5679999999999999e-05 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131118+0000", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020829099999999999 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131138+0000", 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9162999999999999e-05 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134541+0000", 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00340348 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.134624+0000", 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2886999999999994e-05 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45032.0:96811 10.c e205/184 10:3bccbb40:::benchmark_data_smithi042_191918_object96810:head v 205'6011, mlcod=205'6007)", 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 294.808821776, 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.153947004, 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983406+0000", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983405+0000", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:54.983453+0000", 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8809e-05 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131021+0000", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14756804700000001 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131029+0000", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1760000000000004e-06 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131066+0000", 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7713999999999998e-05 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.131085+0000", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9096000000000001e-05 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.137207+0000", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061217750000000003 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:55.137353+0000", 2024-09-13T14:52:49.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014560300000000001 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:41057 11.5 e259/244 11:a8709d99:::benchmark_data_smithi042_223320_object41056:head v 259'2548, mlcod=259'2546)", 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:51:51.224432+0000", 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 58.567795365999999, 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.122995116, 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224432+0000", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224432+0000", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224431+0000", 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.867 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224453+0000", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1770999999999999e-05 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224464+0000", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0531e-05 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224470+0000", 2024-09-13T14:52:49.868 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.314e-06 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224518+0000", 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8035999999999999e-05 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224538+0000", 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9623000000000002e-05 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.347374+0000", 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.122836572 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.347427+0000", 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3088999999999999e-05 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:41056 11.0 e259/244 11:043e7263:::benchmark_data_smithi042_223320_object41055:head v 259'2559, mlcod=259'2557)", 2024-09-13T14:52:49.870 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:51:51.224540+0000", 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 58.567687761000002, 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.123945263, 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224540+0000", 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224540+0000", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224539+0000", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224584+0000", 2024-09-13T14:52:49.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4838e-05 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224593+0000", 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6789999999999994e-06 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.224598+0000", 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2669999999999997e-06 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.343552+0000", 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.11895415400000001 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.343574+0000", 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1611999999999999e-05 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.348449+0000", 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048754089999999998 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.348485+0000", 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5853999999999999e-05 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:41071 11.5 e259/244 11:a98ee389:::benchmark_data_smithi042_223320_object41070:head v 259'2549, mlcod=259'2546)", 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:51:51.232302+0000", 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 58.559925196999998, 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.115151607, 2024-09-13T14:52:49.875 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.232302+0000", 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.232302+0000", 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.232301+0000", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.232329+0000", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7979000000000001e-05 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.343483+0000", 2024-09-13T14:52:49.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.111154352 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.343501+0000", 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7737999999999999e-05 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.344841+0000", 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013396370000000001 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.344858+0000", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7138000000000001e-05 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.347432+0000", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0025736380000000001 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:51:51.347454+0000", 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2399e-05 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.879 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:80398 11.3 e262/244 11:c0150a7d:::benchmark_data_smithi042_223320_object80397:head v 262'5050, mlcod=262'5048)", 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:21.116803+0000", 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 28.675424689, 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.22529660400000001, 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116803+0000", 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116803+0000", 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116802+0000", 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116825+0000", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2959000000000002e-05 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116836+0000", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0920999999999999e-05 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116842+0000", 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8090000000000004e-06 2024-09-13T14:52:49.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116876+0000", 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3055000000000001e-05 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116890+0000", 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4926000000000001e-05 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.120532+0000", 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036418010000000001 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342099+0000", 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.221567187 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:80400 11.e e262/244 11:7b99693c:::benchmark_data_smithi042_223320_object80399:head v 262'5099, mlcod=262'5097)", 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:21.116877+0000", 2024-09-13T14:52:49.884 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 28.675350434999999, 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.225251857, 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116877+0000", 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116877+0000", 2024-09-13T14:52:49.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116877+0000", 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116897+0000", 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0251999999999999e-05 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116904+0000", 2024-09-13T14:52:49.888 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1879999999999996e-06 2024-09-13T14:52:49.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116910+0000", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1330000000000004e-06 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116951+0000", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1066000000000003e-05 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.116972+0000", 2024-09-13T14:52:49.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0574e-05 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.120547+0000", 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035753669999999999 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342129+0000", 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.221581735 2024-09-13T14:52:49.890 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:80404 11.5 e262/244 11:a9e844ef:::benchmark_data_smithi042_223320_object80403:head v 262'4934, mlcod=262'4932)", 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:21.117934+0000", 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 28.674293791, 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.22420996100000001, 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117934+0000", 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117934+0000", 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117933+0000", 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117958+0000", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4927999999999999e-05 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117968+0000", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0628000000000001e-05 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.117975+0000", 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7039999999999997e-06 2024-09-13T14:52:49.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.118038+0000", 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2365000000000003e-05 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.118056+0000", 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8487000000000001e-05 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.120682+0000", 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026263160000000001 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342144+0000", 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.22146136 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:80407 11.0 e262/244 11:0934b1ab:::benchmark_data_smithi042_223320_object80406:head v 262'5020, mlcod=262'5018)", 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:21.124830+0000", 2024-09-13T14:52:49.895 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 28.66739755, 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.22231806700000001, 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.124830+0000", 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.124830+0000", 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.124829+0000", 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.124891+0000", 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1952999999999995e-05 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.897 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.341937+0000", 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21704633700000001 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.341952+0000", 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4416999999999999e-05 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342125+0000", 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017328299999999999 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342152+0000", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6658000000000001e-05 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.347113+0000", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0049612440000000001 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.347148+0000", 2024-09-13T14:52:49.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5258e-05 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:80431 11.5 e262/244 11:a457845f:::benchmark_data_smithi042_223320_object80430:head v 262'4935, mlcod=262'4932)", 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:52:21.148832+0000", 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 28.643395239, 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.199044321, 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.148832+0000", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.148832+0000", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.148831+0000", 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:52:49.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.148858+0000", 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7191000000000001e-05 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.341938+0000", 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19307971800000001 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:52:49.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.341951+0000", 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3407000000000001e-05 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342174+0000", 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022299700000000001 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.342188+0000", 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3457e-05 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.347839+0000", 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.005651026 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:52:21.347877+0000", 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7985000000000003e-05 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:52:49.904 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:52:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:49.906 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:49.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:49.916 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:49] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-09-13T14:52:50.007 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:50.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.055 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:50.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:50.156 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:50.184 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:50.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:50.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.329 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:50.480 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:52:50.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:50.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:50.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:52:50.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.725 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:50.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:50.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:52:50.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:52:50.779143+0000 min lat: 0.00116536 max lat: 2.04401 avg lat: 0.0116493 2024-09-13T14:52:50.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:52:50.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 109714 109698 85.6786 45.25 0.00360396 0.0116493 2024-09-13T14:52:50.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:50.854 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:50.956 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:50.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:50.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:51.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:51.109 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:51.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:51.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:51.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.374 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:51.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:51.505 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:52:51.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:51.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:51.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:51 smithi159 ceph-mon[102519]: pgmap v920: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-09-13T14:52:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:51 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:51 smithi159 ceph-mon[102519]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 110468 110452 85.2026 47.125 0.0196117 0.0117313 2024-09-13T14:52:51.813 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:51.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[108570]: pgmap v920: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[108570]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[113926]: pgmap v920: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-09-13T14:52:51.817 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:52:51.818 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/560029237' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:52:51.818 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[113926]: osdmap e268: 8 total, 8 up, 8 in 2024-09-13T14:52:51.818 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:52:51.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:51.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:51.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:52.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:52.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:52.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.242 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:52.308 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:52.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.453 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:52.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:52.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:52.577 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:52.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:52.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:52.604 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:52:52.605 INFO:teuthology.orchestra.run.smithi042.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:52:52.605 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:52:52.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:52.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:52.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:52.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 111379 111363 84.8579 56.9375 0.0292341 0.011779 2024-09-13T14:52:52.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:52.886 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:52.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:52.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:53.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:53.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:53.148 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:53.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:53.250 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:53.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:53.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:53.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:52:53.393 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:53.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:53.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:53.554 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:53.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:53.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:53.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:52:53.705 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:53 smithi042 ceph-mon[113926]: pgmap v922: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2024-09-13T14:52:53.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:53 smithi042 ceph-mon[108570]: pgmap v922: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2024-09-13T14:52:53.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:53.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:52:53.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:53 smithi159 ceph-mon[102519]: pgmap v922: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 918 op/s 2024-09-13T14:52:53.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 112338 112322 84.5577 59.9375 0.0117876 0.0118212 2024-09-13T14:52:53.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:53.873 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:53.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:53.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:53.975 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.005 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:54.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:54.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:54.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:54.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:54.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:54.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:54.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:54.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:54.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:54.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:54.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 113319 113303 84.2809 61.3125 0.00708225 0.0118595 2024-09-13T14:52:54.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:54.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:54.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:54.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:54.970 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:54.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:54.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:55.072 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.104 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:55.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:55.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:55.235 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:55.337 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:55.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:55.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:55.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:55 smithi042 ceph-mon[113926]: pgmap v923: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-09-13T14:52:55.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:55 smithi042 ceph-mon[108570]: pgmap v923: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-09-13T14:52:55.609 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.648 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:55.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:55.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:55 smithi159 ceph-mon[102519]: pgmap v923: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-09-13T14:52:55.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:55.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 114328 114312 84.0313 63.0625 0.0202888 0.0118948 2024-09-13T14:52:55.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:55.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:55.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:55.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:56.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:56.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:56.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:56.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:56.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:56.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:56.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:56.460 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:56.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:56.647 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:56.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:56.749 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:56.775 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:56.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:56.779 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 115311 115295 83.7685 61.4375 0.033073 0.0119324 2024-09-13T14:52:56.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:56.913 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:56.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.014 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:57.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:57.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:57.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:57.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:57.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:57.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:57.458 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:57.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:57.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:57.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:57.659 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:57 smithi042 ceph-mon[113926]: pgmap v924: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:52:57.660 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:57 smithi042 ceph-mon[108570]: pgmap v924: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:52:57.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:57.737 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:57.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:57 smithi159 ceph-mon[102519]: pgmap v924: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s 2024-09-13T14:52:57.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 116242 116226 83.4745 58.1875 0.00552458 0.0119743 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:116266 11.6 11:6e183be0:::benchmark_data_smithi042_223320_object116265:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:57.805895+0000", 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.02898684, 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.029021569000000001, 2024-09-13T14:52:57.837 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 116266 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805895+0000", 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.838 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805895+0000", 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805897+0000", 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7439999999999999e-06 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:52:57.839 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805934+0000", 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6402000000000001e-05 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805950+0000", 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5947000000000002e-05 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.805959+0000", 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2299999999999997e-06 2024-09-13T14:52:57.840 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.806189+0000", 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023040499999999999 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.806326+0000", 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000136904 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.841 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.806465+0000", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013895800000000001 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.814908+0000", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0084431699999999998 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.818736+0000", 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038280380000000002 2024-09-13T14:52:57.842 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:116268 11.6 11:6c739bbb:::benchmark_data_smithi042_223320_object116267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:57.809896+0000", 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.024986323000000001, 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.025101195, 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:52:57.843 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 116268 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809896+0000", 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.844 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809896+0000", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809898+0000", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6559999999999999e-06 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809923+0000", 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5199e-05 2024-09-13T14:52:57.845 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809938+0000", 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4977000000000001e-05 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.809947+0000", 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0680000000000003e-06 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.846 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.810789+0000", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00084223700000000002 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.810968+0000", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017905199999999999 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.811111+0000", 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000142466 2024-09-13T14:52:57.847 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.815164+0000", 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040535049999999998 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.818957+0000", 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003792639 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:52:57.848 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:116280 11.6 11:631927be:::benchmark_data_smithi042_223320_object116279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:57.823700+0000", 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.011182565, 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011370296, 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:57.849 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 116280 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823700+0000", 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823700+0000", 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.850 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823701+0000", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.629e-06 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823723+0000", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2007000000000001e-05 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823736+0000", 2024-09-13T14:52:57.851 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2675e-05 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823750+0000", 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4098e-05 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.823968+0000", 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021757600000000001 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.852 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.824336+0000", 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00036854800000000001 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.824754+0000", 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041817599999999998 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:52:57.853 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829375+0000", 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046201669999999997 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:116287 11.6 11:619ebc23:::benchmark_data_smithi042_223320_object116286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:57.829641+0000", 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0052410110000000003, 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0054971179999999996, 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:52:57.854 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 116287 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829641+0000", 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.855 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829641+0000", 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829643+0000", 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7749999999999999e-06 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.856 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829670+0000", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.739e-05 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829690+0000", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9269000000000001e-05 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829698+0000", 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6489999999999997e-06 2024-09-13T14:52:57.857 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829761+0000", 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2570000000000006e-05 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.829896+0000", 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013548299999999999 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:52:57.858 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.830028+0000", 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132087 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:57.834102+0000", 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040736330000000001 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 4 2024-09-13T14:52:57.859 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:52:57.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:57.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:57.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:57.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.049 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:58.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.151 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:58.184 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:58.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:52:58.330 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:52:58.331 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:52:58.343 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:58.354 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:52:58.354 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:52:58.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:58.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:58.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:52:58.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:52:58.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:58.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:58.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:58.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3527665315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:58.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 117177 117161 83.19 58.4375 0.00583075 0.0120148 2024-09-13T14:52:58.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:52:58.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3527665315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:58.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3527665315' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:52:58.876 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:58.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:58.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:52:58.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:52:58.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:52:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:59.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:52:59.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:52:59.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:59.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:52:59.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.304 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:52:59.304 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:52:59.304 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T14:52:59.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd out 4 2024-09-13T14:52:59.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:59.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:59.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:59.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:52:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:52:59 smithi042 ceph-mon[113926]: pgmap v925: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:52:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:52:59 smithi042 ceph-mon[108570]: pgmap v925: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:52:59.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:52:59.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:52:59.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:52:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:52:59 smithi159 ceph-mon[102519]: pgmap v925: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:52:59.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:59.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:52:59.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 15 118078 118063 82.8887 56.375 0.0258648 0.0120584 2024-09-13T14:52:59.862 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:52:59.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:52:59.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:52:59.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:52:59.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:52:59.963 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:52:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:52:59] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-13T14:53:00.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:00.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:00.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:00.146 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:00.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:00.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:00.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:00.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:00.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:00.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:00.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:00.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:00.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0042 2024-09-13T14:53:00.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 119003 2024-09-13T14:53:00.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:53:00.780 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 82.6371 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 37.4693 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 213.438 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 0 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1322 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 599.508 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3415 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 0 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0120968 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0273068 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 2.04401 2024-09-13T14:53:00.781 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00116536 2024-09-13T14:53:00.784 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:53:00.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:00.858 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:00.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:00.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:00.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:01.096 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:01.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:01.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:01.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:01.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:01.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:01.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:01.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.676 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:01.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:01.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:01 smithi159 ceph-mon[102519]: pgmap v926: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:53:01.749 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:01 smithi042 ceph-mon[113926]: pgmap v926: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:53:01.750 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:01 smithi042 ceph-mon[108570]: pgmap v926: 99 pgs: 99 active+clean; 7.0 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s 2024-09-13T14:53:01.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:01.820 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:01.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:01.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:01.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:02.053 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:02.086 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:02.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:02.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.214 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:02.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:02.315 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:02.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:02.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:02.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:02.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:02.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:02.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:02.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:02.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:02.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:02.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:02.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:03.037 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:03.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:03.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.316 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:03.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:03.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:03.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:03.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:03.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:03.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:03 smithi159 ceph-mon[102519]: pgmap v927: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-13T14:53:03.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:03.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:03 smithi042 ceph-mon[113926]: pgmap v927: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-13T14:53:03.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:03 smithi042 ceph-mon[108570]: pgmap v927: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-13T14:53:03.872 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:03.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:03.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:04.013 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:04.115 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.152 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:04.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:04.253 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:04.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:04.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:04.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:04.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:04.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:04.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:04.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:04.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:04.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:05.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:05.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:05.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:05.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:05.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:05.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:05.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:05.514 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:05.551 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:05.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:05.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:05.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:05.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:05.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:05.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:05.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:05 smithi042 ceph-mon[113926]: pgmap v928: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 50 MiB/s wr, 881 op/s 2024-09-13T14:53:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:05 smithi042 ceph-mon[108570]: pgmap v928: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 50 MiB/s wr, 881 op/s 2024-09-13T14:53:05.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:05.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:05.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:05.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:05.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:05 smithi159 ceph-mon[102519]: pgmap v928: 99 pgs: 99 active+clean; 7.2 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 50 MiB/s wr, 881 op/s 2024-09-13T14:53:06.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:06.089 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:06.191 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:06.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:06.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:06.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:06.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:06.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:06.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:06.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:06.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:06.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.644 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:06.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.724 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:06.725 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:06.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:06.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:06.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:06.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:06.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:07.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.114 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:07.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:07.216 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:07.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:07.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:07.479 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:07.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:07.527 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:53:07.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[113926]: pgmap v929: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 941 op/s 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3637435970' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[108570]: pgmap v929: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 941 op/s 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3637435970' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:07.733 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:07 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:07.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:07.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:07.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:07.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:07.901 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:07.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:07.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:07 smithi159 ceph-mon[102519]: pgmap v929: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 941 op/s 2024-09-13T14:53:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3637435970' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:07 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:08.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:08.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:08.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:08.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:08.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:08.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:08.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:08.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:08.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:08.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:08.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:08.602 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.629 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:08.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:08.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.756 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:08.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:08.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:53:08.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[108570]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T14:53:08.770+0000 7fcad2a32640 -1 mon.a@0(leader).osd e269 definitely_dead 0 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:08.800 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:08 smithi042 ceph-mon[113926]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:08.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:53:08.858 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:08.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:08.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:08.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:08.986 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:08 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:53:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:08 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:53:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:08 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:08 smithi159 ceph-mon[102519]: osdmap e269: 8 total, 8 up, 8 in 2024-09-13T14:53:09.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:09.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:09.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.157 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:09.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:09.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.295 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:09.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:09.369 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:09.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:09.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:09.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:09.553 INFO:teuthology.orchestra.run.smithi042.stderr:marked out osd.4. 2024-09-13T14:53:09.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.618 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:09.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:09.663 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[113926]: pgmap v931: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 951 op/s 2024-09-13T14:53:09.663 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3599485053' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.663 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.663 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[113926]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:53:09.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[108570]: pgmap v931: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 951 op/s 2024-09-13T14:53:09.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3599485053' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.664 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-mon[108570]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:53:09.720 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:09.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:09.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:09.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.864 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:09.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:09.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:09.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:09.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:09.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:09 smithi159 ceph-mon[102519]: pgmap v931: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 28 MiB/s wr, 951 op/s 2024-09-13T14:53:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3599485053' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-13T14:53:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:09 smithi159 ceph-mon[102519]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-13T14:53:10.031 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:09] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-13T14:53:10.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:10.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:10.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:10.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.364 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:10.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:10.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:10.715 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:10.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:10.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:10.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:10.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:53:10.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[108570]: osdmap e270: 8 total, 8 up, 7 in 2024-09-13T14:53:10.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[108570]: osdmap e271: 8 total, 8 up, 7 in 2024-09-13T14:53:10.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:53:10.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[113926]: osdmap e270: 8 total, 8 up, 7 in 2024-09-13T14:53:10.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:10 smithi042 ceph-mon[113926]: osdmap e271: 8 total, 8 up, 7 in 2024-09-13T14:53:10.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:10.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:10.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:10.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:10.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:10 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:10 smithi159 ceph-mon[102519]: osdmap e270: 8 total, 8 up, 7 in 2024-09-13T14:53:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:10 smithi159 ceph-mon[102519]: osdmap e271: 8 total, 8 up, 7 in 2024-09-13T14:53:11.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:11.121 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:11.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:11.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:11.351 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.377 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:11.402 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:53:11.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:11.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:11.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.630 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:11.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:11.755 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:11 smithi042 ceph-mon[108570]: pgmap v933: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 4.6 MiB/s wr, 758 op/s 2024-09-13T14:53:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:11 smithi042 ceph-mon[108570]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:53:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:11 smithi042 ceph-mon[113926]: pgmap v933: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 4.6 MiB/s wr, 758 op/s 2024-09-13T14:53:11.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:11 smithi042 ceph-mon[113926]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:53:11.857 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:11.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:11.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:11.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:11 smithi159 ceph-mon[102519]: pgmap v933: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 4.6 MiB/s wr, 758 op/s 2024-09-13T14:53:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:11 smithi159 ceph-mon[102519]: osdmap e272: 8 total, 8 up, 7 in 2024-09-13T14:53:12.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:12.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.145 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.285 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.386 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:12.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.442 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:12.543 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:12.575 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:12.596 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:12.683 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:53:12.683 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:12.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:53:12.719 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:12.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.766 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:53:12.766 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:12.800 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:53:12.801 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:12.829 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:53:12.829 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:12.830 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:12 smithi042 ceph-mon[113926]: 11.c starting backfill to osd.7 from (0'0,0'0] MAX to 269'7990 2024-09-13T14:53:12.830 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:12 smithi042 ceph-mon[113926]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:53:12.830 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:12 smithi042 ceph-mon[108570]: 11.c starting backfill to osd.7 from (0'0,0'0] MAX to 269'7990 2024-09-13T14:53:12.830 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:12 smithi042 ceph-mon[108570]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:53:12.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:12.862 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:53:12.863 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:12.907 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:12.907 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:53:12.908 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:12.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:12.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:12.955 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:53:12.956 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:12 smithi159 ceph-mon[102519]: 11.c starting backfill to osd.7 from (0'0,0'0] MAX to 269'7990 2024-09-13T14:53:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:12 smithi159 ceph-mon[102519]: osdmap e273: 8 total, 8 up, 7 in 2024-09-13T14:53:13.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:13.048 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:13.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:13.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:13.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:13.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:13.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:13.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:13.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:13.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:13.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:13.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:13.611 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:13.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:13.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:13.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[108570]: pgmap v936: 99 pgs: 1 activating+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 15 KiB/s rd, 45 KiB/s wr, 702 op/s; 7403/332997 objects degraded (2.223%); 33 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-13T14:53:13.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:53:13.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1233454523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:13.775 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[108570]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:53:13.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[113926]: pgmap v936: 99 pgs: 1 activating+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 15 KiB/s rd, 45 KiB/s wr, 702 op/s; 7403/332997 objects degraded (2.223%); 33 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-13T14:53:13.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:53:13.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1233454523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:13.776 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:13 smithi042 ceph-mon[113926]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:53:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:13.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:13.878 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:13.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:13.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:13 smithi159 ceph-mon[102519]: pgmap v936: 99 pgs: 1 activating+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.8 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 15 KiB/s rd, 45 KiB/s wr, 702 op/s; 7403/332997 objects degraded (2.223%); 33 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-13T14:53:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:13 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T14:53:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1233454523' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:13 smithi159 ceph-mon[102519]: osdmap e274: 8 total, 8 up, 7 in 2024-09-13T14:53:14.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:14.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:14.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:14.153 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:14.180 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:14.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:14.306 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:14.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:14.408 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:14.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:14.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:14.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:14.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:14.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:14.726 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:14.769 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:14.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:14.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:14.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:14.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:14.892 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:14 smithi042 ceph-mon[113926]: 1.5 scrub starts 2024-09-13T14:53:14.892 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:14 smithi042 ceph-mon[113926]: 1.5 scrub ok 2024-09-13T14:53:14.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:14 smithi042 ceph-mon[108570]: 1.5 scrub starts 2024-09-13T14:53:14.892 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:14 smithi042 ceph-mon[108570]: 1.5 scrub ok 2024-09-13T14:53:14.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:14 smithi159 ceph-mon[102519]: 1.5 scrub starts 2024-09-13T14:53:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:14 smithi159 ceph-mon[102519]: 1.5 scrub ok 2024-09-13T14:53:15.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:15.101 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:15.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:15.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:15.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:15.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:15.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:15.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:15.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:15.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:31166 11.6 11:6370d035:::benchmark_data_smithi042_223320_object31165:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.055296+0000", 2024-09-13T14:53:15.592 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.532606126999994, 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.101287033, 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 31166 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055296+0000", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055296+0000", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055297+0000", 2024-09-13T14:53:15.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.302e-06 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055318+0000", 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1421999999999999e-05 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055329+0000", 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0531e-05 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055336+0000", 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3900000000000004e-06 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055405+0000", 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8393999999999996e-05 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055536+0000", 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013087299999999999 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055666+0000", 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013000699999999999 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.059064+0000", 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033981279999999998 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.059079+0000", 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5206e-05 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156462+0000", 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.097382641000000006 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156532+0000", 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0865999999999995e-05 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156583+0000", 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0272999999999998e-05 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:31171 11.6 11:6f2df096:::benchmark_data_smithi042_223320_object31170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.055881+0000", 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.532021197000006, 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.101052088, 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 31171 2024-09-13T14:53:15.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055881+0000", 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055881+0000", 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055881+0000", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5199999999999995e-07 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055901+0000", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0000000000000002e-05 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055912+0000", 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0477e-05 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.055919+0000", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9619999999999996e-06 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.057022+0000", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001103571 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.057129+0000", 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000106671 2024-09-13T14:53:15.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.057252+0000", 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012297600000000001 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.060539+0000", 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003287248 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.060952+0000", 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041230800000000001 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156753+0000", 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.095801839999999999 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156840+0000", 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6787999999999996e-05 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.156933+0000", 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2394999999999999e-05 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:31173 11.6 11:66dd8e9f:::benchmark_data_smithi042_223320_object31172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.058116+0000", 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.529785752999999, 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.100436495, 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.606 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.607 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.607 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 31173 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058116+0000", 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058116+0000", 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058118+0000", 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5489999999999999e-06 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058139+0000", 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1382e-05 2024-09-13T14:53:15.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058151+0000", 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2099000000000001e-05 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.058159+0000", 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.174e-06 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.059103+0000", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00094412300000000005 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.059213+0000", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109678 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.059343+0000", 2024-09-13T14:53:15.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013009799999999999 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.060899+0000", 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001555968 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.061066+0000", 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016678299999999999 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.158457+0000", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.097391538999999999 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.158514+0000", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6184999999999998e-05 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.158553+0000", 2024-09-13T14:53:15.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8917e-05 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45314.0:31220 11.2 e259/248 11:48026359:::benchmark_data_smithi042_223320_object31219:head v 259'1976, mlcod=259'1973)", 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.180968+0000", 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.406934227999997, 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.134945388, 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.180968+0000", 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.180968+0000", 2024-09-13T14:53:15.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.180968+0000", 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5099999999999999e-07 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.181024+0000", 2024-09-13T14:53:15.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5835999999999999e-05 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.181035+0000", 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0929e-05 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.181042+0000", 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3749999999999997e-06 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.181097+0000", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4676999999999998e-05 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.181117+0000", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.012e-05 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.182376+0000", 2024-09-13T14:53:15.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0012590430000000001 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315913+0000", 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.133536657 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45314.0:31227 11.1 e259/248 11:83d63877:::benchmark_data_smithi042_223320_object31226:head v 259'1901, mlcod=259'1898)", 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.182675+0000", 2024-09-13T14:53:15.619 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.405227085999996, 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13929108900000001, 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.182675+0000", 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.182675+0000", 2024-09-13T14:53:15.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.182676+0000", 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6800000000000009e-07 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.182714+0000", 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8216000000000001e-05 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315883+0000", 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13316871799999999 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315895+0000", 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2241e-05 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315955+0000", 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9982000000000003e-05 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315976+0000", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1636000000000001e-05 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.321924+0000", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0059478990000000004 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.321966+0000", 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1428999999999999e-05 2024-09-13T14:53:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45314.0:31237 11.b e259/248 11:d0fdbeeb:::benchmark_data_smithi042_223320_object31236:head v 259'1941, mlcod=259'1938)", 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:45.185468+0000", 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 90.402434064000005, 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13265486700000001, 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.185468+0000", 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.185468+0000", 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.185469+0000", 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2190000000000001e-06 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.185498+0000", 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9357999999999999e-05 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315882+0000", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.130383375 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315892+0000", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0798999999999999e-05 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315958+0000", 2024-09-13T14:53:15.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5123999999999995e-05 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.315977+0000", 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9862999999999999e-05 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.318069+0000", 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020916340000000002 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:45.318123+0000", 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3495000000000002e-05 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:33626 11.6 11:6a1fa5aa:::benchmark_data_smithi042_223320_object33625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:46.707977+0000", 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 88.879924881999997, 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.095145624999999998, 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 33626 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.707977+0000", 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.707977+0000", 2024-09-13T14:53:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.707979+0000", 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.736e-06 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708000+0000", 2024-09-13T14:53:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1271e-05 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708019+0000", 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8652999999999999e-05 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708025+0000", 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8499999999999996e-06 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708083+0000", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7945999999999997e-05 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708223+0000", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014010300000000001 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.708377+0000", 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015387699999999999 2024-09-13T14:53:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.709621+0000", 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0012433190000000001 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.709783+0000", 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016199300000000001 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.802959+0000", 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.093176629999999996 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.803046+0000", 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7046999999999995e-05 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:46.803123+0000", 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6199999999999995e-05 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:34815 11.6 11:6f0ca10f:::benchmark_data_smithi042_223320_object34814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:47.511690+0000", 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 88.076211623000006, 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17706221799999999, 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 34815 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511690+0000", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511690+0000", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511692+0000", 2024-09-13T14:53:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.519e-06 2024-09-13T14:53:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511718+0000", 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6486e-05 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511733+0000", 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4460999999999999e-05 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.511741+0000", 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2619999999999998e-06 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.513222+0000", 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014807279999999999 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.513356+0000", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013389399999999999 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.513599+0000", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024381699999999999 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.519096+0000", 2024-09-13T14:53:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0054963709999999999 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.521801+0000", 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027055830000000001 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.688656+0000", 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16685520600000001 2024-09-13T14:53:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.688709+0000", 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2856999999999999e-05 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.688752+0000", 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3034000000000002e-05 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:34831 11.6 11:6e9a44ec:::benchmark_data_smithi042_223320_object34830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:47.520294+0000", 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 88.067607967000001, 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.170053922, 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 34831 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520294+0000", 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520294+0000", 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520295+0000", 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.145e-06 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520402+0000", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107404 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520414+0000", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1235e-05 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520422+0000", 2024-09-13T14:53:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1300000000000001e-06 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520499+0000", 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6802000000000001e-05 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520614+0000", 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011505 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.520695+0000", 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1260000000000005e-05 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.521929+0000", 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001233811 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.522351+0000", 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000422099 2024-09-13T14:53:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690143+0000", 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.167792578 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690298+0000", 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015442400000000001 2024-09-13T14:53:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690348+0000", 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9984000000000001e-05 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:34848 11.6 11:6b449aac:::benchmark_data_smithi042_223320_object34847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:47.560212+0000", 2024-09-13T14:53:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 88.027689968999994, 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13060131699999999, 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 34848 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560212+0000", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560212+0000", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560214+0000", 2024-09-13T14:53:15.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4849999999999999e-06 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560239+0000", 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4723e-05 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560260+0000", 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0979e-05 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560269+0000", 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3899999999999999e-06 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560351+0000", 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1742000000000004e-05 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560491+0000", 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000140333 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.560632+0000", 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014076 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.562144+0000", 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015113889999999999 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.562534+0000", 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039065200000000001 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690389+0000", 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.127855149 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690539+0000", 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149579 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:47.690813+0000", 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00027413600000000002 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.660 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:45219 11.6 11:64ff9f07:::benchmark_data_smithi042_223320_object45218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.813525+0000", 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.774377143999999, 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17109310799999999, 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 45219 2024-09-13T14:53:15.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813525+0000", 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813525+0000", 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813526+0000", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.55e-06 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813567+0000", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0352999999999998e-05 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813579+0000", 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2442999999999999e-05 2024-09-13T14:53:15.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813587+0000", 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1340000000000004e-06 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.816892+0000", 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033051510000000001 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.817093+0000", 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020045499999999999 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.817197+0000", 2024-09-13T14:53:15.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010409299999999999 2024-09-13T14:53:15.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.819801+0000", 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026042320000000002 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.669 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.819877+0000", 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6031000000000004e-05 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863976+0000", 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.044098831999999998 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984564+0000", 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.120587877 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984618+0000", 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3956999999999999e-05 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:45220 11.6 11:6c2f8763:::benchmark_data_smithi042_223320_object45219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.813600+0000", 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.774301687999994, 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17115997999999999, 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.672 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 45220 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813600+0000", 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813600+0000", 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813601+0000", 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1500000000000004e-07 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813622+0000", 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1155e-05 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813630+0000", 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4339999999999992e-06 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.813636+0000", 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1190000000000002e-06 2024-09-13T14:53:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.818640+0000", 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.005003998 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.818764+0000", 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000123363 2024-09-13T14:53:15.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.818877+0000", 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000112933 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.819925+0000", 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010482 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.677 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.820371+0000", 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00044578199999999998 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984675+0000", 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16430420200000001 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984726+0000", 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0630000000000001e-05 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.678 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984760+0000", 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4548999999999997e-05 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:45226 11.6 11:60b7c0b6:::benchmark_data_smithi042_223320_object45225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e259)", 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.858647+0000", 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.729254945999998, 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12630786999999999, 2024-09-13T14:53:15.679 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 45226 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858647+0000", 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.680 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858647+0000", 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858649+0000", 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5849999999999998e-06 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.681 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858678+0000", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8909000000000001e-05 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858697+0000", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8284999999999999e-05 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858706+0000", 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6860000000000001e-06 2024-09-13T14:53:15.682 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858782+0000", 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5504000000000001e-05 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.858935+0000", 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015289899999999999 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.859085+0000", 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000150054 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.862847+0000", 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037625200000000001 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984807+0000", 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12196012000000001 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.684 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984850+0000", 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2645000000000003e-05 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984918+0000", 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7550000000000002e-05 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.685 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984955+0000", 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7113000000000002e-05 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45314.0:45219 11.6 e259/248 ondisk, result = 0)", 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.863873+0000", 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.724028509999997, 2024-09-13T14:53:15.686 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.120755073, 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863873+0000", 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.687 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863873+0000", 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863874+0000", 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0699999999999997e-07 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863878+0000", 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4429999999999996e-06 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.688 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863888+0000", 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7599999999999997e-06 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863895+0000", 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5599999999999999e-06 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.689 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863963+0000", 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.792e-05 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.863970+0000", 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6450000000000005e-06 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984628+0000", 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.120658138 2024-09-13T14:53:15.690 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45314.0:45228 11.1 e259/248 11:8024a303:::benchmark_data_smithi042_223320_object45227:head v 259'2716, mlcod=259'2714)", 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.864141+0000", 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.723760475000006, 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12187445099999999, 2024-09-13T14:53:15.691 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.692 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.864141+0000", 2024-09-13T14:53:15.693 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.693 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.693 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.693 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.693 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.864141+0000", 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.694 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.864142+0000", 2024-09-13T14:53:15.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3399999999999998e-07 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.695 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.864182+0000", 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0102999999999999e-05 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984048+0000", 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.119865598 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.696 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984065+0000", 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6685e-05 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984221+0000", 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015685300000000001 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.697 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984246+0000", 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4354000000000001e-05 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.985884+0000", 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001638635 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.986016+0000", 2024-09-13T14:53:15.698 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013138900000000001 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45314.0:45220 11.6 e259/248 ondisk, result = 0)", 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:51:53.869718+0000", 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 81.718184231999999, 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.115053146, 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:53:15.699 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.869718+0000", 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.869718+0000", 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.700 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.869718+0000", 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4900000000000001e-07 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.869721+0000", 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3100000000000001e-06 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.701 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984057+0000", 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.11433618299999999 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984068+0000", 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0084999999999999e-05 2024-09-13T14:53:15.702 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984662+0000", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059471700000000003 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984669+0000", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1380000000000001e-06 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:51:53.984771+0000", 2024-09-13T14:53:15.703 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101264 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:76515 11.6 11:6ba84164:::benchmark_data_smithi042_223320_object76514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e262)", 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:17.385888+0000", 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 58.202014105000003, 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.110898596, 2024-09-13T14:53:15.704 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 76515 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385888+0000", 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.705 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385888+0000", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385890+0000", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.289e-06 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385918+0000", 2024-09-13T14:53:15.706 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7718000000000001e-05 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385936+0000", 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8547999999999999e-05 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.707 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.385945+0000", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9420000000000001e-06 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.386630+0000", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00068457499999999998 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.386757+0000", 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012768999999999999 2024-09-13T14:53:15.708 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.386898+0000", 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014056999999999999 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.390887+0000", 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039894420000000002 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.709 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.394810+0000", 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039224610000000004 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.496496+0000", 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.10168572400000001 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.496743+0000", 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024770599999999999 2024-09-13T14:53:15.710 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:17.496786+0000", 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2930999999999999e-05 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:90525 11.6 11:6f9b4eb7:::benchmark_data_smithi042_223320_object90524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:30.529572+0000", 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 45.058329819999997, 2024-09-13T14:53:15.711 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12017254500000001, 2024-09-13T14:53:15.712 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.712 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:15 smithi042 ceph-mon[113926]: pgmap v939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 748 op/s; 14887/330627 objects degraded (4.503%); 6783/330627 objects misplaced (2.052%); 1.1 MiB/s, 2 keys/s, 21 objects/s recovering 2024-09-13T14:53:15.712 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:15 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 14887/330627 objects degraded (4.503%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:53:15.712 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:15 smithi042 ceph-mon[108570]: pgmap v939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 748 op/s; 14887/330627 objects degraded (4.503%); 6783/330627 objects misplaced (2.052%); 1.1 MiB/s, 2 keys/s, 21 objects/s recovering 2024-09-13T14:53:15.713 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:15 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 14887/330627 objects degraded (4.503%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 90525 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.713 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529572+0000", 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529572+0000", 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.714 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529574+0000", 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.007e-06 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529597+0000", 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3164e-05 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529612+0000", 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5183e-05 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.715 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529621+0000", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6100000000000006e-06 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529742+0000", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120881 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529863+0000", 2024-09-13T14:53:15.716 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012110800000000001 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.529971+0000", 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.536614+0000", 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0066427430000000004 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.717 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.536649+0000", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5125e-05 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.649373+0000", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.11272444199999999 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.649598+0000", 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000224873 2024-09-13T14:53:15.718 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.649744+0000", 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014640899999999999 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:90527 11.6 11:6e4cafd4:::benchmark_data_smithi042_223320_object90526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:30.530096+0000", 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 45.057806176, 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12796654499999999, 2024-09-13T14:53:15.719 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 90527 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530096+0000", 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.720 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530096+0000", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530097+0000", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.057e-06 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530121+0000", 2024-09-13T14:53:15.721 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4221000000000001e-05 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530134+0000", 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3118e-05 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.530142+0000", 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3000000000000002e-06 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.722 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.533970+0000", 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038274089999999999 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.534063+0000", 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3734999999999996e-05 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.534153+0000", 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9192999999999998e-05 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.723 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.536787+0000", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026339699999999998 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.536983+0000", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019607899999999999 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.657556+0000", 2024-09-13T14:53:15.724 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.120573149 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.657779+0000", 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022271899999999999 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:30.658062+0000", 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00028359499999999998 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.725 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:91450 11.6 11:69606dcb:::benchmark_data_smithi042_223320_object91449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:52:31.504672+0000", 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 44.083229310999997, 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.282991247, 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 91450 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.726 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504672+0000", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504672+0000", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504674+0000", 2024-09-13T14:53:15.727 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9319999999999999e-06 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504694+0000", 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9792999999999998e-05 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504710+0000", 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5361000000000001e-05 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.728 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504718+0000", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6139999999999992e-06 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504785+0000", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.635e-05 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.504985+0000", 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000200316 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.729 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.505116+0000", 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013101200000000001 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.508576+0000", 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003460435 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.730 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.508772+0000", 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000195737 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.787542+0000", 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.27876999000000002 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:53:15.731 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.787610+0000", 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7762000000000006e-05 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:52:31.787664+0000", 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3944999999999999e-05 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:53:15.732 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:53:15.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:15.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:15.889 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:53:15.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T14:53:15.907 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:15.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:15 smithi159 ceph-mon[102519]: pgmap v939: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 2 remapped+peering, 93 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 748 op/s; 14887/330627 objects degraded (4.503%); 6783/330627 objects misplaced (2.052%); 1.1 MiB/s, 2 keys/s, 21 objects/s recovering 2024-09-13T14:53:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:15 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 14887/330627 objects degraded (4.503%), 1 pg degraded (PG_DEGRADED) 2024-09-13T14:53:16.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:16.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:16.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:16.133 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.155 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.316 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:16.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:16.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:16.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:16.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:16.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:16.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:16.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:16.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:16.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:16.963 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:16 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:53:16.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:16.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:16.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:17.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:16 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:53:17.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:16 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T14:53:17.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:17.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:17.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.272 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:17.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:17.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:17.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:17.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:17.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:17.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:17.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:17.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:17.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:17.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:17.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:17.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:17.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:17.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:17.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:17 smithi159 ceph-mon[102519]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 94 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 15014/327273 objects degraded (4.588%); 13546/327273 objects misplaced (4.139%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2024-09-13T14:53:18.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:17 smithi042 ceph-mon[108570]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 94 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 15014/327273 objects degraded (4.588%); 13546/327273 objects misplaced (4.139%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2024-09-13T14:53:18.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:17 smithi042 ceph-mon[113926]: pgmap v940: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 activating+undersized+remapped, 94 active+clean; 6.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 676 op/s; 15014/327273 objects degraded (4.588%); 13546/327273 objects misplaced (4.139%); 1.3 MiB/s, 1 keys/s, 23 objects/s recovering 2024-09-13T14:53:18.181 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:18.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:18.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.332 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:18.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:18.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:18.569 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:18.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:18.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:18.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:18.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:18.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:18.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:18.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:18.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:19.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:19.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:19.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:19.230 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:19.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.377 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:19.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:19.479 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:19.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:19.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:19.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:19.761 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:19.788 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:19.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:19.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:19.915 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:19.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:19.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:19 smithi159 ceph-mon[102519]: pgmap v941: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15014/323976 objects degraded (4.634%); 13546/323976 objects misplaced (4.181%); 1.1 MiB/s, 19 objects/s recovering 2024-09-13T14:53:20.017 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:20.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:19 smithi042 ceph-mon[113926]: pgmap v941: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15014/323976 objects degraded (4.634%); 13546/323976 objects misplaced (4.181%); 1.1 MiB/s, 19 objects/s recovering 2024-09-13T14:53:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:19 smithi042 ceph-mon[108570]: pgmap v941: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15014/323976 objects degraded (4.634%); 13546/323976 objects misplaced (4.181%); 1.1 MiB/s, 19 objects/s recovering 2024-09-13T14:53:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:19] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-13T14:53:20.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:20.196 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:20.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:20.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:20.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:20.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:20.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:20.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:20.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:20.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:20.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:20.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:20.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:20.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:20.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:20.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:20.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15014/323976 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:21.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15014/323976 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:21.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15014/323976 objects degraded (4.634%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:21.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:21.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.265 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:21.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:21.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:21.531 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:21.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:21.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:21.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:21.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:21.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:21.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:21.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:21.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:22.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[113926]: pgmap v942: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15014/321588 objects degraded (4.669%); 13340/321588 objects misplaced (4.148%); 964 KiB/s, 15 objects/s recovering 2024-09-13T14:53:22.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:22.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:22.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[108570]: pgmap v942: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15014/321588 objects degraded (4.669%); 13340/321588 objects misplaced (4.148%); 964 KiB/s, 15 objects/s recovering 2024-09-13T14:53:22.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:22.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.068 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:22.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:22.102 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:53:22.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:22.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:21 smithi159 ceph-mon[102519]: pgmap v942: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 472 op/s; 15014/321588 objects degraded (4.669%); 13340/321588 objects misplaced (4.148%); 964 KiB/s, 15 objects/s recovering 2024-09-13T14:53:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:22.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:22.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:22.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:22.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:22.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:22.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:22.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:22.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:22.809 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:22.886 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:22.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:22.913 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:22.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:22.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:22.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:23.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:23.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:23.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.180 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:23.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.282 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:23.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:23.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:23.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:23.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:23.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:23.798 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:53:23.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:23.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:23.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:23.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:23.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:23.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:24.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[113926]: pgmap v943: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15136/314949 objects degraded (4.806%); 13145/314949 objects misplaced (4.174%); 1.2 MiB/s, 20 objects/s recovering 2024-09-13T14:53:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1481987035' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[108570]: pgmap v943: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15136/314949 objects degraded (4.806%); 13145/314949 objects misplaced (4.174%); 1.2 MiB/s, 20 objects/s recovering 2024-09-13T14:53:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1481987035' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:23 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:24.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:23 smithi159 ceph-mon[102519]: pgmap v943: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15136/314949 objects degraded (4.806%); 13145/314949 objects misplaced (4.174%); 1.2 MiB/s, 20 objects/s recovering 2024-09-13T14:53:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:23 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1481987035' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:23 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:53:24.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:24.476 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:24.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:24.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:24.618 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:24.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:24.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:24.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:24.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:24.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:24.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:24.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:24.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:25.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:25.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:25.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[113926]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:53:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:25.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:24 smithi042 ceph-mon[108570]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:53:25.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:25.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:25.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:24 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:24 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:53:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:24 smithi159 ceph-mon[102519]: osdmap e275: 8 total, 8 up, 7 in 2024-09-13T14:53:25.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:25.396 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:25.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:25.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:25.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:25.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:25.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:25.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:25.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:25.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:25.989 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:25.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:25 smithi042 ceph-mon[113926]: pgmap v945: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 15136/311808 objects degraded (4.854%); 12956/311808 objects misplaced (4.155%); 797 KiB/s, 12 objects/s recovering 2024-09-13T14:53:26.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15136/311808 objects degraded (4.854%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:25 smithi042 ceph-mon[108570]: pgmap v945: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 15136/311808 objects degraded (4.854%); 12956/311808 objects misplaced (4.155%); 797 KiB/s, 12 objects/s recovering 2024-09-13T14:53:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15136/311808 objects degraded (4.854%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:26.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:26.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.215 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:25 smithi159 ceph-mon[102519]: pgmap v945: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 15136/311808 objects degraded (4.854%); 12956/311808 objects misplaced (4.155%); 797 KiB/s, 12 objects/s recovering 2024-09-13T14:53:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15136/311808 objects degraded (4.854%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:26.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:26.362 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:26.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:26.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.488 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:26.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:26.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:26.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:26.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:26.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:26.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:26.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:26.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:26.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:27.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:27.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:27.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.155 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:27.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:27.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:27.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:27.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:27.440 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:27.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:27.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:27.567 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:27.669 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:27.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:27.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.832 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:27.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:27.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:27.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.035 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:27 smithi042 ceph-mon[113926]: pgmap v946: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15290/308337 objects degraded (4.959%); 12743/308337 objects misplaced (4.133%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:28.036 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:27 smithi042 ceph-mon[108570]: pgmap v946: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15290/308337 objects degraded (4.959%); 12743/308337 objects misplaced (4.133%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:28.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:28.087 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:28.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:28.213 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:27 smithi159 ceph-mon[102519]: pgmap v946: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 15290/308337 objects degraded (4.959%); 12743/308337 objects misplaced (4.133%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:28.315 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:28.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:28.445 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:28.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:28.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:28.735 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:53:28.739 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:28.818 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T14:53:28.841 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:28.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:28.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:28.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:28.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3136700750' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:29.033 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:29.034 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3136700750' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:29.034 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:29.034 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3136700750' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:29.034 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T14:53:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:29.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:29.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:29.282 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.310 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.412 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:29.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:29.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:29.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:29.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:29.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:29.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:29.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:29.857 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:29.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:29.956 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[113926]: pgmap v947: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 15290/304806 objects degraded (5.016%); 12743/304806 objects misplaced (4.181%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:29.957 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:29.957 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[113926]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:53:29.957 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[108570]: pgmap v947: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 15290/304806 objects degraded (5.016%); 12743/304806 objects misplaced (4.181%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:29.957 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:29.957 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-mon[108570]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:53:29.958 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:29] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.51.0" 2024-09-13T14:53:29.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:29.982 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:29.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:30.054 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:53:30.055 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:30.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.090 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:53:30.090 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:30.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:30.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:30.157 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:53:30.157 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:30.210 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.211 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:53:30.212 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:30.244 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:53:30.244 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:29 smithi159 ceph-mon[102519]: pgmap v947: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 15290/304806 objects degraded (5.016%); 12743/304806 objects misplaced (4.181%); 844 KiB/s, 13 objects/s recovering 2024-09-13T14:53:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T14:53:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:29 smithi159 ceph-mon[102519]: osdmap e276: 8 total, 8 up, 7 in 2024-09-13T14:53:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.264 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:30.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.288 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:53:30.288 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:30.322 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:53:30.322 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:30.354 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:53:30.354 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:30.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:30.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:30.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:30.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:30.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:30.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T14:53:30.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:30.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:30.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:30.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:30.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:30.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:30.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:31.032 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:31.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:30 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1562075858' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15290/304806 objects degraded (5.016%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:30 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1562075858' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15290/304806 objects degraded (5.016%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:31.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:31.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:30 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1562075858' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15290/304806 objects degraded (5.016%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:31.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:31.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.329 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:31.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:31.423 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:31.424 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:31.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:31.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:31.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:31.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:31.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:31.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:31.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:31.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:31.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:31.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:31.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:31.990 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:31.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:32.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:31 smithi042 ceph-mon[108570]: pgmap v949: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 15290/302427 objects degraded (5.056%); 12542/302427 objects misplaced (4.147%); 472 KiB/s, 7 objects/s recovering 2024-09-13T14:53:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:31 smithi042 ceph-mon[113926]: pgmap v949: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 15290/302427 objects degraded (5.056%); 12542/302427 objects misplaced (4.147%); 472 KiB/s, 7 objects/s recovering 2024-09-13T14:53:32.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:32.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:32.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:32.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:31 smithi159 ceph-mon[102519]: pgmap v949: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 6.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 15290/302427 objects degraded (5.056%); 12542/302427 objects misplaced (4.147%); 472 KiB/s, 7 objects/s recovering 2024-09-13T14:53:32.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:32.389 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:32.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:32.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:32.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:32.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:32.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:32.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:32.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.811 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:32.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:32.912 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:32.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:32.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:32.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:33.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:33.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:33.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:33.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:33.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:33.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:33.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:33.445 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:33.485 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:33.587 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:33.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:33.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:33.714 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:33.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:33.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:33.879 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:33.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:33.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:33.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:34.010 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:33 smithi042 ceph-mon[113926]: pgmap v950: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 15445/295548 objects degraded (5.226%); 12328/295548 objects misplaced (4.171%); 876 KiB/s, 13 objects/s recovering 2024-09-13T14:53:34.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:33 smithi042 ceph-mon[108570]: pgmap v950: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 15445/295548 objects degraded (5.226%); 12328/295548 objects misplaced (4.171%); 876 KiB/s, 13 objects/s recovering 2024-09-13T14:53:34.112 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.154 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:34.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:33 smithi159 ceph-mon[102519]: pgmap v950: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 6.0 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 15445/295548 objects degraded (5.226%); 12328/295548 objects misplaced (4.171%); 876 KiB/s, 13 objects/s recovering 2024-09-13T14:53:34.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:34.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:34.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:34.319 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:34.319 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:34.387 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:34.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:34.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.615 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:34.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.764 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:34.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:34.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:34.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:34.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:34.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:53:35.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:35.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:35.161 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:34 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:35.178 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:35.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:34 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:35.186 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:34 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:53:35.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.305 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:35.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:35.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:35.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:35.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.863 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:35.954 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[113926]: pgmap v951: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 15445/292467 objects degraded (5.281%); 12128/292467 objects misplaced (4.147%); 755 KiB/s, 11 objects/s recovering 2024-09-13T14:53:35.954 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15445/292467 objects degraded (5.281%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[108570]: pgmap v951: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 15445/292467 objects degraded (5.281%); 12128/292467 objects misplaced (4.147%); 755 KiB/s, 11 objects/s recovering 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15445/292467 objects degraded (5.281%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:35.955 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:35.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:35.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:35.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:35.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:36.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:53:36.089 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.171 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:36.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:36.201 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:35 smithi159 ceph-mon[102519]: pgmap v951: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 644 op/s; 15445/292467 objects degraded (5.281%); 12128/292467 objects misplaced (4.147%); 755 KiB/s, 11 objects/s recovering 2024-09-13T14:53:36.201 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15445/292467 objects degraded (5.281%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:36.202 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:36.202 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:36.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:36.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:36.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:36.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:36.543 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:36.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:36.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:36.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:36.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:36.850 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:36.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:36.953 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:36.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:36.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:36.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:37.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:37.103 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:37.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:37.205 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:37.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:37.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:37.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.480 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:37.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.582 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:37.604 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:37.706 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:37.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:37.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:37.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:37.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:37.870 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:37.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:37.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:38.015 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:38.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:38.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:38.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:37 smithi159 ceph-mon[102519]: pgmap v952: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 15619/288618 objects degraded (5.412%); 11899/288618 objects misplaced (4.123%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:38.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:38.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:37 smithi042 ceph-mon[108570]: pgmap v952: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 15619/288618 objects degraded (5.412%); 11899/288618 objects misplaced (4.123%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:38.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:37 smithi042 ceph-mon[113926]: pgmap v952: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 15619/288618 objects degraded (5.412%); 11899/288618 objects misplaced (4.123%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:38.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:38.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:38.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:38.561 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:38.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:38.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:38.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:38.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:38.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:38.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:38.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:38.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:39.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:39.121 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:39.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:39.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.304 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:39.351 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:39.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.379 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:39.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:39.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.531 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:39.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:39.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:39.661 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:39.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:39.763 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:39.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.792 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:39.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:39.807 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:39] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:53:39.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:39.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:39.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:39.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:40.023 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:40.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:40.074 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[113926]: pgmap v953: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15619/284862 objects degraded (5.483%); 11899/284862 objects misplaced (4.177%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:40.075 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.075 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.075 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[108570]: pgmap v953: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15619/284862 objects degraded (5.483%); 11899/284862 objects misplaced (4.177%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:40.075 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.076 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:40.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:40.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:39 smithi159 ceph-mon[102519]: pgmap v953: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 664 op/s; 15619/284862 objects degraded (5.483%); 11899/284862 objects misplaced (4.177%); 729 KiB/s, 11 objects/s recovering 2024-09-13T14:53:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:53:40.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:40.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:40.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:40.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:40.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:40.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:40.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:40.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:40.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:40.882 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:53:40.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:40.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:40.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:41.012 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15619/284862 objects degraded (5.483%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:41.012 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:41.014 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15619/284862 objects degraded (5.483%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:41.014 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:41.023 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:41.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:41.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:41.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:41.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15619/284862 objects degraded (5.483%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:53:41.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:41.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:41.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:41.557 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:41.580 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:41.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:41.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:41.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:41.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:41.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:41.980 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:41.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:42.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:42.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:42.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:42.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: pgmap v954: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 15619/282276 objects degraded (5.533%); 11665/282276 objects misplaced (4.132%); 628 KiB/s, 9 objects/s recovering 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/246979953' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.212 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: pgmap v954: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 15619/282276 objects degraded (5.533%); 11665/282276 objects misplaced (4.132%); 628 KiB/s, 9 objects/s recovering 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/246979953' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.213 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:41 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: pgmap v954: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 15619/282276 objects degraded (5.533%); 11665/282276 objects misplaced (4.132%); 628 KiB/s, 9 objects/s recovering 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1246513893' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: osdmap e277: 8 total, 8 up, 7 in 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/246979953' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:41 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T14:53:42.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:42.353 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.393 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:42.495 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:42.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:42.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:42.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:42.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:42.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:42.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:42.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:42.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:42.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:43.030 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:53:43.030 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:53:43.030 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:42 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:43.030 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:42 smithi042 ceph-mon[113926]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:53:43.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:42 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:43.031 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:42 smithi042 ceph-mon[108570]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:53:43.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:43.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:43.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:42 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T14:53:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:42 smithi159 ceph-mon[102519]: osdmap e278: 8 total, 8 up, 7 in 2024-09-13T14:53:43.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:43.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:43.362 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:43.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.404 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:43.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:43.505 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:43.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:43.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:43.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:43.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:43.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:43.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:43.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:43.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:43.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:44.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:44.057 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:43 smithi042 ceph-mon[113926]: pgmap v957: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15745/275892 objects degraded (5.707%); 11473/275892 objects misplaced (4.159%); 968 KiB/s, 15 objects/s recovering 2024-09-13T14:53:44.057 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:43 smithi042 ceph-mon[108570]: pgmap v957: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15745/275892 objects degraded (5.707%); 11473/275892 objects misplaced (4.159%); 968 KiB/s, 15 objects/s recovering 2024-09-13T14:53:44.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:44.100 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:53:44.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:53:44.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:44.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:43 smithi159 ceph-mon[102519]: pgmap v957: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 690 op/s; 15745/275892 objects degraded (5.707%); 11473/275892 objects misplaced (4.159%); 968 KiB/s, 15 objects/s recovering 2024-09-13T14:53:44.257 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T14:53:44.303 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:44.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:44.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:44.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:44.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:44.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:44.604 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:44.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:44.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:44.748 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:44.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:44.896 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:44.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:44.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:45.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T14:53:45.065 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:45.131 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:45.165 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:45.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.296 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:45.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:45.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.482 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:45.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:45.583 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:45.628 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:45.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:45.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:45.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:45.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:45.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:45.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:45.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:45.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:46.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:46.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:46.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:46.175 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:45 smithi042 ceph-mon[108570]: pgmap v958: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 15745/272850 objects degraded (5.771%); 11291/272850 objects misplaced (4.138%); 528 KiB/s, 8 objects/s recovering 2024-09-13T14:53:46.176 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15745/272850 objects degraded (5.771%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:46.176 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:45 smithi042 ceph-mon[113926]: pgmap v958: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 15745/272850 objects degraded (5.771%); 11291/272850 objects misplaced (4.138%); 528 KiB/s, 8 objects/s recovering 2024-09-13T14:53:46.176 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15745/272850 objects degraded (5.771%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:46.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:46.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:45 smithi159 ceph-mon[102519]: pgmap v958: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.6 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 15745/272850 objects degraded (5.771%); 11291/272850 objects misplaced (4.138%); 528 KiB/s, 8 objects/s recovering 2024-09-13T14:53:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15745/272850 objects degraded (5.771%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:46.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.305 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:46.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:46.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:46.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:46.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:46.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:46.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:46.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:46.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:46.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:46.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:46.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.850 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:46.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:46.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:46.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:46.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:46.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:47.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:47.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:47.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:47.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:47.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:47.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:47.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:47.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:47.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:47.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.650 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:47.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:47.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.794 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:47.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:47.895 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:47.934 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:47.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:47.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:48.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:48.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:48.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:48.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:48.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:47 smithi159 ceph-mon[102519]: pgmap v959: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15885/269355 objects degraded (5.897%); 11090/269355 objects misplaced (4.117%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:48.304 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:47 smithi042 ceph-mon[113926]: pgmap v959: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15885/269355 objects degraded (5.897%); 11090/269355 objects misplaced (4.117%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:48.304 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:47 smithi042 ceph-mon[108570]: pgmap v959: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 15885/269355 objects degraded (5.897%); 11090/269355 objects misplaced (4.117%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:48.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.324 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:48.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:48.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:48.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:48.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.652 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:48.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:48.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:48.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:48.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:48.789 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:53:48.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:48.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:48.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:49.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:49.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:49.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:49.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:49.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:49.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:49.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:49.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:49.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:49.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:49.771 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:49] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:53:49.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:49.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:49.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:49.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:49.974 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:53:49.974 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:53:49.975 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:53:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:49.985 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:49.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:50.050 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:53:50.051 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:53:50.051 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:49 smithi042 ceph-mon[113926]: pgmap v960: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 15885/265644 objects degraded (5.980%); 11090/265644 objects misplaced (4.175%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:50.051 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:49 smithi042 ceph-mon[108570]: pgmap v960: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 15885/265644 objects degraded (5.980%); 11090/265644 objects misplaced (4.175%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:50.086 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.089 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:53:50.090 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:53:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.126 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:53:50.126 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:53:50.178 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:53:50.179 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:53:50.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:50.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:53:50.224 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:53:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:49 smithi159 ceph-mon[102519]: pgmap v960: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 15885/265644 objects degraded (5.980%); 11090/265644 objects misplaced (4.175%); 1015 KiB/s, 15 objects/s recovering 2024-09-13T14:53:50.260 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:53:50.261 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:53:50.299 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:53:50.299 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:53:50.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:50.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:50.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.594 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:50.719 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:50.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:50.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:50.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:50.910 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:50.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:51.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:51.037 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:51.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:51.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1130594005' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15885/265644 objects degraded (5.980%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:51.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1130594005' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15885/265644 objects degraded (5.980%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:51.153 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:51.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1130594005' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2222109387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: osdmap e279: 8 total, 8 up, 7 in 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15885/265644 objects degraded (5.980%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:53:51.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:51.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:51.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:51.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:51.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:51.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:51.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:51.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:51.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:51.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:51.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:51.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:51.876 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:51.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:51.978 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:52.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:52.059 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:51 smithi042 ceph-mon[108570]: pgmap v962: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 15885/263295 objects degraded (6.033%); 10918/263295 objects misplaced (4.147%); 457 KiB/s, 7 objects/s recovering 2024-09-13T14:53:52.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:51 smithi042 ceph-mon[113926]: pgmap v962: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 15885/263295 objects degraded (6.033%); 10918/263295 objects misplaced (4.147%); 457 KiB/s, 7 objects/s recovering 2024-09-13T14:53:52.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:52.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:52.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:51 smithi159 ceph-mon[102519]: pgmap v962: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 491 op/s; 15885/263295 objects degraded (6.033%); 10918/263295 objects misplaced (4.147%); 457 KiB/s, 7 objects/s recovering 2024-09-13T14:53:52.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:52.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:52.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.457 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:52.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:52.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:52.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:52.682 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:52.710 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:52.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:52.812 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:52.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:52.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:52.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:52.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:53.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:53.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:53.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:53.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:53.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:53.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:53.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:53.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:53.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:53.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:53.636 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:53.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:53.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:53.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:53.886 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:53.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:53.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:53.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:54.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:54.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:54.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:54.226 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:53 smithi159 ceph-mon[102519]: pgmap v963: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/256758 objects degraded (6.241%); 10714/256758 objects misplaced (4.173%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:54.327 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:53 smithi042 ceph-mon[113926]: pgmap v963: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/256758 objects degraded (6.241%); 10714/256758 objects misplaced (4.173%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:54.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:53 smithi042 ceph-mon[108570]: pgmap v963: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/256758 objects degraded (6.241%); 10714/256758 objects misplaced (4.173%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:54.351 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:54.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:54.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:54.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.604 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:54.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:54.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:54.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:54.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:54.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:54.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:54.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:54.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:55.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:55.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.152 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:55.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:55.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:55.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:55.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:55.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:53:55.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:53:55.520 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:55.542 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:55.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:55.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:55.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:55.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:55.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:55.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:55.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:55.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:56.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:56.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:53:56.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:56.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:53:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:55 smithi159 ceph-mon[102519]: pgmap v964: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/253731 objects degraded (6.315%); 10485/253731 objects misplaced (4.132%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16024/253731 objects degraded (6.315%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:56.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:56.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:56.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:55 smithi042 ceph-mon[113926]: pgmap v964: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/253731 objects degraded (6.315%); 10485/253731 objects misplaced (4.132%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16024/253731 objects degraded (6.315%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:55 smithi042 ceph-mon[108570]: pgmap v964: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 16024/253731 objects degraded (6.315%); 10485/253731 objects misplaced (4.132%); 806 KiB/s, 12 objects/s recovering 2024-09-13T14:53:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16024/253731 objects degraded (6.315%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:53:56.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:56.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:56.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:56.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:56.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:56.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:56.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:56.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:56.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:56.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:56.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:57.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:57.128 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:57.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:57.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:57.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:57.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:57.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:57.562 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:57.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:53:57.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:57.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:53:57.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:57.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:57.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:57.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:57.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:57.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:57.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:58.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:53:57 smithi042 ceph-mon[113926]: pgmap v965: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 16125/250179 objects degraded (6.445%); 10296/250179 objects misplaced (4.115%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:53:58.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:53:57 smithi042 ceph-mon[108570]: pgmap v965: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 16125/250179 objects degraded (6.445%); 10296/250179 objects misplaced (4.115%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:53:58.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:58.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:58.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:58.243 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:58.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:53:57 smithi159 ceph-mon[102519]: pgmap v965: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 16125/250179 objects degraded (6.445%); 10296/250179 objects misplaced (4.115%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:53:58.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:58.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:58.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:58.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:58.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:58.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:58.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:58.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:53:58.826 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:58.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:53:58.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:58.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:58.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:58.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:59.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:59.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:53:59.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:59.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:53:59.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:59.412 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:59.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:59.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:59.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:53:59.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:53:59.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:53:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:53:59.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:53:59.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:59.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:53:59.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:53:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:53:59] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.51.0" 2024-09-13T14:53:59.876 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:53:59.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:53:59.949 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:53:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:53:59.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:00.051 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:00.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.078 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:00.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:00.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.186 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:00 smithi042 ceph-mon[108570]: pgmap v966: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 16125/246360 objects degraded (6.545%); 10296/246360 objects misplaced (4.179%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:54:00.186 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:00 smithi042 ceph-mon[113926]: pgmap v966: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 16125/246360 objects degraded (6.545%); 10296/246360 objects misplaced (4.179%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:54:00.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:00.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.403 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:00.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:00 smithi159 ceph-mon[102519]: pgmap v966: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 5.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 16125/246360 objects degraded (6.545%); 10296/246360 objects misplaced (4.179%); 979 KiB/s, 15 objects/s recovering 2024-09-13T14:54:00.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65369 10.12 10:4b9c46a1:::benchmark_data_smithi042_191918_object65368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.508 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.168268028, 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6915967300000001, 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65369 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.509 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334281+0000", 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6199999999999998e-07 2024-09-13T14:54:00.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334305+0000", 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4281e-05 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334314+0000", 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0529999999999996e-06 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334322+0000", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9959999999999993e-06 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334933+0000", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00061030700000000004 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334939+0000", 2024-09-13T14:54:00.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.601e-06 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.005813+0000", 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.670873692 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.005962+0000", 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149417 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.006134+0000", 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017209199999999999 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010130+0000", 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039954209999999999 2024-09-13T14:54:00.514 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016394+0000", 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062645540000000003 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.024165+0000", 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0077701740000000004 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025313+0000", 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011481270000000001 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025350+0000", 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7178000000000002e-05 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025877+0000", 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00052737499999999996 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65380 10.12 10:4b19149e:::benchmark_data_smithi042_191918_object65379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16821655400003, 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.697573869, 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.517 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65380 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334332+0000", 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334333+0000", 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3799999999999997e-07 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334354+0000", 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1263999999999998e-05 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334362+0000", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3629999999999994e-06 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334368+0000", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8850000000000004e-06 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334964+0000", 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059618500000000005 2024-09-13T14:54:00.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334971+0000", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8560000000000001e-06 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008063+0000", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6730918909999999 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008164+0000", 2024-09-13T14:54:00.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100981 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.008268+0000", 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010392500000000001 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010281+0000", 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020129570000000001 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019809+0000", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0095286329999999999 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025983+0000", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006173825 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.030493+0000", 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045095300000000003 2024-09-13T14:54:00.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.030567+0000", 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4090000000000004e-05 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.031906+0000", 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0013389459999999999 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.524 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65390 10.12 10:4d7a0b2a:::benchmark_data_smithi042_191918_object65389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16817107399999, 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6988079219999999, 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.525 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65390 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334377+0000", 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334378+0000", 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3900000000000004e-07 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334403+0000", 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5208999999999999e-05 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334411+0000", 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9189999999999999e-06 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334417+0000", 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4240000000000001e-06 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335000+0000", 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00058312299999999995 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335006+0000", 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6000000000000003e-06 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009095+0000", 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6740885250000002 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009217+0000", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000121959 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.009319+0000", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101903 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012251+0000", 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029319609999999999 2024-09-13T14:54:00.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019953+0000", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0077021370000000004 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.028546+0000", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.008593481 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.031984+0000", 2024-09-13T14:54:00.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034381390000000002 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.032052+0000", 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7909000000000003e-05 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.033185+0000", 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001132994 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.532 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65423 10.12 10:4e8869da:::benchmark_data_smithi042_191918_object65422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16811323799999, 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7009449569999999, 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65423 2024-09-13T14:54:00.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334435+0000", 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334436+0000", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5200000000000002e-07 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334457+0000", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1718000000000001e-05 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334466+0000", 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6430000000000001e-06 2024-09-13T14:54:00.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334472+0000", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0020000000000003e-06 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335032+0000", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00055971700000000005 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335038+0000", 2024-09-13T14:54:00.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3180000000000003e-06 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010303+0000", 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6752650099999999 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010390+0000", 2024-09-13T14:54:00.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6410999999999996e-05 2024-09-13T14:54:00.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.010488+0000", 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8797999999999998e-05 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.013187+0000", 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026985580000000002 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020042+0000", 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068551489999999996 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.028612+0000", 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0085700380000000003 2024-09-13T14:54:00.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034471+0000", 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058586389999999997 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034542+0000", 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1489999999999995e-05 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.541 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.035380+0000", 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00083801399999999999 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65445 10.12 10:4aecfbce:::benchmark_data_smithi042_191918_object65444:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16806703899999, 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7099828289999999, 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.542 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65445 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334482+0000", 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-07 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334503+0000", 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1220999999999999e-05 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334512+0000", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5799999999999992e-06 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334518+0000", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2779999999999999e-06 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335060+0000", 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00054184700000000003 2024-09-13T14:54:00.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335066+0000", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9699999999999996e-06 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011296+0000", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.676230428 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011444+0000", 2024-09-13T14:54:00.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014804600000000001 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.011575+0000", 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000131013 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.013340+0000", 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001764926 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020114+0000", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067741720000000002 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034076+0000", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013961341 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.042908+0000", 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088317910000000003 2024-09-13T14:54:00.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.042956+0000", 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8053000000000002e-05 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.044464+0000", 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015086780000000001 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65472 10.12 10:4cbbc5fc:::benchmark_data_smithi042_191918_object65471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16802166500003, 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7122895630000001, 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65472 2024-09-13T14:54:00.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334527+0000", 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6800000000000001e-07 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334554+0000", 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6183999999999999e-05 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334564+0000", 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0068000000000001e-05 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334571+0000", 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2049999999999996e-06 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335092+0000", 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00052121999999999997 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335099+0000", 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5629999999999997e-06 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012291+0000", 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6771921509999999 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012404+0000", 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113379 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.012501+0000", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6994e-05 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.015452+0000", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029508910000000002 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025262+0000", 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098095260000000007 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.034134+0000", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088725550000000007 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.045952+0000", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011818276000000001 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.045993+0000", 2024-09-13T14:54:00.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0343000000000003e-05 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046816+0000", 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00082374 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65476 10.12 10:4d513365:::benchmark_data_smithi042_191918_object65475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:54:00.557 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16780148999999, 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.712834333, 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65476 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334747+0000", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334748+0000", 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5099999999999999e-07 2024-09-13T14:54:00.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334802+0000", 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4203999999999998e-05 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334811+0000", 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8689999999999993e-06 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334818+0000", 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2880000000000004e-06 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335612+0000", 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00079335099999999997 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335620+0000", 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9740000000000008e-06 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016436+0000", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6808159040000001 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016630+0000", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019332099999999999 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.016846+0000", 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021587000000000001 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.019905+0000", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030591199999999998 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.025923+0000", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060186570000000002 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.040837+0000", 2024-09-13T14:54:00.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014913786 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046901+0000", 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060638510000000003 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.046961+0000", 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0019999999999998e-05 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047581+0000", 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00062036700000000003 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:65492 10.12 10:4de5edea:::benchmark_data_smithi042_191918_object65491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e200)", 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 393.16769931900001, 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.713522875, 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.565 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 65492 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334849+0000", 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334850+0000", 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7700000000000004e-07 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334911+0000", 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.156e-05 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334920+0000", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4910000000000005e-06 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.334926+0000", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.635e-06 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335649+0000", 2024-09-13T14:54:00.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00072289499999999998 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:27.335657+0000", 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7079999999999997e-06 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018279+0000", 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6826222849999999 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018444+0000", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164267 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5,6", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.018648+0000", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000204761 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.020003+0000", 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001354102 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.026048+0000", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060454590000000004 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047641+0000", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021593478999999999 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047749+0000", 2024-09-13T14:54:00.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107308 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.047822+0000", 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2855999999999995e-05 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:47:29.048372+0000", 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00055049199999999997 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.572 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124021 10.14 10:2ba3697e:::benchmark_data_smithi042_191918_object428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 338.04974613100001, 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1046680210000002, 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.573 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124021 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452804+0000", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7010000000000001e-06 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452810+0000", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5620000000000003e-06 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452828+0000", 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8241999999999999e-05 2024-09-13T14:54:00.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452836+0000", 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1459999999999996e-06 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453443+0000", 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060669599999999999 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453453+0000", 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0169999999999999e-05 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.376987+0000", 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92353413299999998 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377000+0000", 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3369e-05 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494162+0000", 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.117161158 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494306+0000", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014396600000000001 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.548321+0000", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.054015170000000001 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555032+0000", 2024-09-13T14:54:00.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006711308 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555051+0000", 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9389e-05 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555093+0000", 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1325000000000002e-05 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555161+0000", 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8522999999999994e-05 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557470+0000", 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023091629999999999 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.581 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124045 10.14 10:2fb89687:::benchmark_data_smithi042_191918_object452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:54:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 338.03090750799998, 2024-09-13T14:54:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0906675360000002, 2024-09-13T14:54:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:00.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:00.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124045 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.585 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471642+0000", 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3349999999999999e-06 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471647+0000", 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.955e-06 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471659+0000", 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2055000000000001e-05 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.588 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471666+0000", 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0890000000000002e-06 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471724+0000", 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7751000000000003e-05 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471734+0000", 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3589999999999997e-06 2024-09-13T14:54:00.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377030+0000", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90529651300000002 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377038+0000", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8059999999999995e-06 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552572+0000", 2024-09-13T14:54:00.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1755340999999999 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552679+0000", 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010687 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.553038+0000", 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000359148 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.559991+0000", 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069528999999999997 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560059+0000", 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7787999999999995e-05 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560076+0000", 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7073999999999999e-05 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560136+0000", 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0300999999999997e-05 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562309+0000", 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021724919999999998 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.593 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124123 10.14 10:2f92ce95:::benchmark_data_smithi042_191918_object530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.96366964600003, 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0288850589999998, 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.594 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124123 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538880+0000", 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4339999999999999e-06 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538886+0000", 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.203e-06 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538900+0000", 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4361e-05 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538906+0000", 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.19e-06 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538972+0000", 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5877000000000006e-05 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538985+0000", 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2565000000000001e-05 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377062+0000", 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.83807765400000001 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377069+0000", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8399999999999997e-06 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557527+0000", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.180457504 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557691+0000", 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164358 2024-09-13T14:54:00.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.558112+0000", 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000421579 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565029+0000", 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006916663 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565048+0000", 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9318e-05 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565106+0000", 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7472999999999997e-05 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565161+0000", 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4979000000000003e-05 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567764+0000", 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026030609999999998 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124160 10.14 10:2a1875cd:::benchmark_data_smithi042_191918_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.93366949099999, 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0036828569999998, 2024-09-13T14:54:00.602 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124160 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568881+0000", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.649e-06 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568886+0000", 2024-09-13T14:54:00.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1680000000000003e-06 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568898+0000", 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2198000000000001e-05 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568905+0000", 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568973+0000", 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7587000000000002e-05 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568983+0000", 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0091e-05 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377092+0000", 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.80810927200000005 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377099+0000", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.754e-06 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562371+0000", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.185272007 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562528+0000", 2024-09-13T14:54:00.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015748699999999999 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562968+0000", 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00043982 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570098+0000", 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0071301250000000002 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570121+0000", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2694e-05 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570194+0000", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2587000000000001e-05 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570259+0000", 2024-09-13T14:54:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5026999999999999e-05 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572562+0000", 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023032870000000002 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124175 10.14 10:2bb4c019:::benchmark_data_smithi042_191918_object582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:54:00.610 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.92043389999998, 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.996287562, 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124175 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:54:00.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582116+0000", 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.697e-06 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582122+0000", 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.31e-06 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582135+0000", 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-05 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.625825+0000", 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.04369029 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626947+0000", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011223139999999999 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626958+0000", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.039e-05 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377122+0000", 2024-09-13T14:54:00.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.75016387699999998 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377128+0000", 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.708e-06 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567848+0000", 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.19071927 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.615 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.616 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568046+0000", 2024-09-13T14:54:00.616 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000198648 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568459+0000", 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041277499999999999 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.575988+0000", 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075292850000000001 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.617 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576003+0000", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4636e-05 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576043+0000", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0141000000000001e-05 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576115+0000", 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2324999999999996e-05 2024-09-13T14:54:00.618 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578402+0000", 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022867859999999999 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124203 10.14 10:2df4aadf:::benchmark_data_smithi042_191918_object610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.87081597500003, 2024-09-13T14:54:00.619 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9513996520000001, 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124203 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:54:00.620 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631734+0000", 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.393e-06 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.621 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631739+0000", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8720000000000001e-06 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631751+0000", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2089000000000001e-05 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631758+0000", 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5310000000000004e-06 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631820+0000", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1235999999999994e-05 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631830+0000", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0087e-05 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377151+0000", 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.74532109400000002 2024-09-13T14:54:00.623 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377157+0000", 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.567e-06 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572621+0000", 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1954635090000001 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.624 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572838+0000", 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000217176 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.573258+0000", 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041985200000000002 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578468+0000", 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052102149999999998 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.625 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578487+0000", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9037000000000001e-05 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581071+0000", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025835509999999999 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581132+0000", 2024-09-13T14:54:00.626 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0745000000000002e-05 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583132+0000", 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002000698 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124303 10.14 10:2cafaf1c:::benchmark_data_smithi042_191918_object710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.77996194299999, 2024-09-13T14:54:00.627 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8779041919999999, 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124303 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.628 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722588+0000", 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.159e-06 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.629 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722592+0000", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1330000000000001e-06 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722602+0000", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0475000000000001e-05 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722609+0000", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6880000000000002e-06 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722665+0000", 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5510000000000002e-05 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722675+0000", 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0122000000000001e-05 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377180+0000", 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.65450549800000002 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377187+0000", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7050000000000002e-06 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578628+0000", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2014406769999999 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578774+0000", 2024-09-13T14:54:00.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014649800000000001 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.579165+0000", 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039142000000000002 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.585995+0000", 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068292939999999996 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590672+0000", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046776760000000004 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597892+0000", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072192139999999998 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597941+0000", 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9373000000000003e-05 2024-09-13T14:54:00.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600491+0000", 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025497499999999999 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124330 10.14 10:2826c96e:::benchmark_data_smithi042_191918_object737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.760622678, 2024-09-13T14:54:00.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.867454178, 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124330 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:54:00.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741928+0000", 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9069999999999999e-06 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741934+0000", 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5509999999999996e-06 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741957+0000", 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2571000000000001e-05 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741966+0000", 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6219999999999998e-06 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742027+0000", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1544000000000006e-05 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742037+0000", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0363999999999999e-05 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377210+0000", 2024-09-13T14:54:00.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.63517246999999999 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377217+0000", 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9410000000000001e-06 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583202+0000", 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2059855310000001 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583348+0000", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000145131 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583747+0000", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039933999999999998 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590701+0000", 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069539379999999998 2024-09-13T14:54:00.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590715+0000", 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4312e-05 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606747+0000", 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016031553 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606846+0000", 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9411000000000001e-05 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609380+0000", 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025339920000000001 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124347 10.14 10:2b51d2b1:::benchmark_data_smithi042_191918_object754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.643 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.743130517, 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.852510501, 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124347 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.644 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759419+0000", 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6000000000000004e-07 2024-09-13T14:54:00.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759421+0000", 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7700000000000002e-06 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759428+0000", 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1199999999999996e-06 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759435+0000", 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.336e-06 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759494+0000", 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9657000000000002e-05 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759505+0000", 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0743e-05 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377240+0000", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.61773501500000005 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377247+0000", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5420000000000002e-06 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586075+0000", 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.208828676 2024-09-13T14:54:00.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586227+0000", 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015113099999999999 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586620+0000", 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039343899999999998 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590721+0000", 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041013100000000004 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590746+0000", 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5072e-05 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609450+0000", 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018703280999999999 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609614+0000", 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164084 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.611929+0000", 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023147649999999999 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.651 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124381 10.14 10:2889f54c:::benchmark_data_smithi042_191918_object788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.71249186199998, 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.831978273, 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124381 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:54:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790060+0000", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.774e-06 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790069+0000", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7450000000000001e-06 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790074+0000", 2024-09-13T14:54:00.654 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5140000000000001e-06 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790150+0000", 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5729000000000006e-05 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790163+0000", 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2855e-05 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.655 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377270+0000", 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.58710728099999998 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377276+0000", 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3759999999999996e-06 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588559+0000", 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2112829700000001 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.656 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588744+0000", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018437400000000001 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.589096+0000", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035260100000000002 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590759+0000", 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016620910000000001 2024-09-13T14:54:00.657 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600557+0000", 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097980979999999999 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619838+0000", 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019281598 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.658 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619895+0000", 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6721000000000001e-05 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622035+0000", 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002140028 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.659 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.660 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124511 10.14 10:2e9e45ad:::benchmark_data_smithi042_191918_object918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.57584131099998, 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6976293679999999, 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124511 2024-09-13T14:54:00.661 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926709+0000", 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5600000000000001e-06 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926715+0000", 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0560000000000001e-06 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926732+0000", 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6949e-05 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.663 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926740+0000", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3559999999999993e-06 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926811+0000", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0926000000000005e-05 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926822+0000", 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:54:00.664 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377300+0000", 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.45047793699999999 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377306+0000", 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6189999999999996e-06 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.665 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590780+0000", 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.213474116 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590981+0000", 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000200495 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.591320+0000", 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00033930899999999999 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.666 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597842+0000", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0065215780000000001 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600611+0000", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027691840000000001 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622109+0000", 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021498397999999998 2024-09-13T14:54:00.667 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622172+0000", 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3184999999999999e-05 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624337+0000", 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021641899999999999 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.668 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124519 10.14 10:2894197e:::benchmark_data_smithi042_191918_object926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 337.56676128499998, 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6923704129999999, 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:00.669 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124519 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.670 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935789+0000", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5149999999999999e-06 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935794+0000", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8069999999999999e-06 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935806+0000", 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721999999999999e-05 2024-09-13T14:54:00.671 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935813+0000", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8079999999999999e-06 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935890+0000", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7139999999999997e-05 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935901+0000", 2024-09-13T14:54:00.672 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1178e-05 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377329+0000", 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.44142803000000003 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377336+0000", 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5459999999999997e-06 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.673 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593242+0000", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.215905585 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593400+0000", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015875199999999999 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593797+0000", 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039664300000000002 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600508+0000", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067110399999999997 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600684+0000", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017580100000000001 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624404+0000", 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023720393999999999 2024-09-13T14:54:00.675 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624466+0000", 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2217000000000004e-05 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.628158+0000", 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036912350000000002 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.676 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.677 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.677 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:00.677 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:00.677 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:54:00.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:00.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:00.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:00.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:00.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:00.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:00.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:00.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:01.050 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:01.153 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:01.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:01.278 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:01.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16125/246360 objects degraded (6.545%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2706573529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16125/246360 objects degraded (6.545%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2706573529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:01 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:01.437 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T14:54:01.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:01.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:01 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16125/246360 objects degraded (6.545%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2706573529' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:01 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T14:54:01.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:01.588 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:01.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:01.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:01.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:01.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.872 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:01.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:01.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:01.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:01.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:01.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:02.099 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:02.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:02.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:02.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[113926]: pgmap v967: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 16125/243612 objects degraded (6.619%); 10053/243612 objects misplaced (4.127%); 937 KiB/s, 14 objects/s recovering 2024-09-13T14:54:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[113926]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:54:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[108570]: pgmap v967: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 16125/243612 objects degraded (6.619%); 10053/243612 objects misplaced (4.127%); 937 KiB/s, 14 objects/s recovering 2024-09-13T14:54:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:02 smithi042 ceph-mon[108570]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:54:02.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:02.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:02 smithi159 ceph-mon[102519]: pgmap v967: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 16125/243612 objects degraded (6.619%); 10053/243612 objects misplaced (4.127%); 937 KiB/s, 14 objects/s recovering 2024-09-13T14:54:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T14:54:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:02 smithi159 ceph-mon[102519]: osdmap e280: 8 total, 8 up, 7 in 2024-09-13T14:54:02.501 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:02.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:02.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:02.675 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:02.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:02.822 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:02.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:02.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:02.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:02.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:03.051 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:03.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:03.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:03.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T14:54:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.200 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:03.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:03.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:03.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:03.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:03.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:03.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:03.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:03.732 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:03.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:03.878 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:03.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:03.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:03.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:04.029 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:04.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:04.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:04.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:04 smithi042 ceph-mon[113926]: pgmap v969: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/236994 objects degraded (6.863%); 9846/236994 objects misplaced (4.155%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:04 smithi042 ceph-mon[108570]: pgmap v969: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/236994 objects degraded (6.863%); 9846/236994 objects misplaced (4.155%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:04.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.431 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:04.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:04.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:04 smithi159 ceph-mon[102519]: pgmap v969: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/236994 objects degraded (6.863%); 9846/236994 objects misplaced (4.155%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:04.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:04.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T14:54:04.549 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:04.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:04.617 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:04.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:04.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.729 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:04.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:04.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:04.964 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:04.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:04.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:05.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:05.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.271 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:05.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:05.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.502 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:05.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:05.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:05.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:05.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:05.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:05.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:05.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:05.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:06.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.042 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.143 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:06.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:06.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:06.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:06.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[113926]: pgmap v970: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/233955 objects degraded (6.953%); 9657/233955 objects misplaced (4.128%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16266/233955 objects degraded (6.953%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[108570]: pgmap v970: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/233955 objects degraded (6.953%); 9657/233955 objects misplaced (4.128%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16266/233955 objects degraded (6.953%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:06.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:06.453 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:06.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:06 smithi159 ceph-mon[102519]: pgmap v970: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 16266/233955 objects degraded (6.953%); 9657/233955 objects misplaced (4.128%); 985 KiB/s, 15 objects/s recovering 2024-09-13T14:54:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:06 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16266/233955 objects degraded (6.953%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:06.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:06.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:06.809 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:06.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:06.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:06.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:06.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:06.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:07.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:07.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:07.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:07.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:07.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:07.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:07.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:07.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:07.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:07.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:07.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:07.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:07.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:07.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:07.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:07.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:07.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:08.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:08.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:08.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:08.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:07 smithi159 ceph-mon[102519]: pgmap v971: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 16266/231582 objects degraded (7.024%); 9657/231582 objects misplaced (4.170%); 422 KiB/s, 6 objects/s recovering 2024-09-13T14:54:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:07 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:07 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.253 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[108570]: pgmap v971: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 16266/231582 objects degraded (7.024%); 9657/231582 objects misplaced (4.170%); 422 KiB/s, 6 objects/s recovering 2024-09-13T14:54:08.253 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.253 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[113926]: pgmap v971: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 16266/231582 objects degraded (7.024%); 9657/231582 objects misplaced (4.170%); 422 KiB/s, 6 objects/s recovering 2024-09-13T14:54:08.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:07 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:08.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.384 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:08.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:08.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:08.533 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:08.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:08.644 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:08.647 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:08.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:08.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:08.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:08.841 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:08.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:08.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:08.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:08.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:08.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:09.069 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:09.084 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:09.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:09.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:09.085 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:09.085 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:09.085 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:09.120 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:09.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:09.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:54:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:54:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:09.271 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:09.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:09.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.400 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:09.405 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:54:09.405 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:09.442 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:54:09.443 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:09.484 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:54:09.485 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:09.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:09.526 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:54:09.526 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:09.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:09.571 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:54:09.571 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:09.609 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:54:09.609 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:09.638 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:54:09.639 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:09.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:09.658 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:54:09.658 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:09.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:09.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:09.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:09.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:09.909 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:09] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:54:09.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:09.932 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:09.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:10.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.071 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:10.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:10.197 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:10.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:09 smithi159 ceph-mon[102519]: pgmap v972: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 16356/226821 objects degraded (7.211%); 9478/226821 objects misplaced (4.179%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1305444544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:10.299 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:09 smithi042 ceph-mon[113926]: pgmap v972: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 16356/226821 objects degraded (7.211%); 9478/226821 objects misplaced (4.179%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:10.299 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1305444544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:10.299 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:09 smithi042 ceph-mon[108570]: pgmap v972: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 650 op/s; 16356/226821 objects degraded (7.211%); 9478/226821 objects misplaced (4.179%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:10.299 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1305444544' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:10.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:10.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:10.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:10.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:10.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:10.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:10.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:10.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:10.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:10.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:10.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:10.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:11.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:11.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.138 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:11.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:11.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:11.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16356/226821 objects degraded (7.211%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:11.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:11.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:11.298 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16356/226821 objects degraded (7.211%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:11.298 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16356/226821 objects degraded (7.211%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T14:54:11.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:11.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:11.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:11.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:11.524 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:11.626 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:11.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:11.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:11.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:11.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:11.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:11.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:11.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:11.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:12.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:12.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:12.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:12.182 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:11 smithi042 ceph-mon[108570]: pgmap v973: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16356/224658 objects degraded (7.280%); 9297/224658 objects misplaced (4.138%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:12.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:11 smithi042 ceph-mon[113926]: pgmap v973: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16356/224658 objects degraded (7.280%); 9297/224658 objects misplaced (4.138%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:12.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:12.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:11 smithi159 ceph-mon[102519]: pgmap v973: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 631 op/s; 16356/224658 objects degraded (7.280%); 9297/224658 objects misplaced (4.138%); 991 KiB/s, 15 objects/s recovering 2024-09-13T14:54:12.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:12.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:12.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:12.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:12.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:12.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.835 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:12.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:12.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:12.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:12.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:13.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.109 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:13.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:13.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.235 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:13.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:13.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:13.370 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:13.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:13.472 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:13.600 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:13.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:13.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:13.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:13.751 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:13.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:13.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:13.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:13.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:13.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:13.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:14.015 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:13 smithi042 ceph-mon[108570]: pgmap v974: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 16518/217977 objects degraded (7.578%); 9074/217977 objects misplaced (4.163%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:14.015 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:13 smithi042 ceph-mon[113926]: pgmap v974: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 16518/217977 objects degraded (7.578%); 9074/217977 objects misplaced (4.163%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:14.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:14.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:14.137 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:13 smithi159 ceph-mon[102519]: pgmap v974: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 16518/217977 objects degraded (7.578%); 9074/217977 objects misplaced (4.163%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:14.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:14.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:14.442 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:14.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:14.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:14.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:14.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:14.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:14.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:14.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:14.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:14.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.846 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:14.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:14.935 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:54:14.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:14.975 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:14.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:15.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.120 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:15.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:15.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2886148471' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:15.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2886148471' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2886148471' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:15.369 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:15.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:15.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:15.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:15.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:15.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:15.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:15.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:15.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:15.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:15.911 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:15.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:15.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:16.013 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.037 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:16.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:15 smithi159 ceph-mon[102519]: pgmap v975: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 16518/214836 objects degraded (7.689%); 8880/214836 objects misplaced (4.133%); 799 KiB/s, 12 objects/s recovering 2024-09-13T14:54:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:15 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:15 smithi159 ceph-mon[102519]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:54:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16518/214836 objects degraded (7.689%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:16.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:16.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:16.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[108570]: pgmap v975: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 16518/214836 objects degraded (7.689%); 8880/214836 objects misplaced (4.133%); 799 KiB/s, 12 objects/s recovering 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[108570]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16518/214836 objects degraded (7.689%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[113926]: pgmap v975: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 16518/214836 objects degraded (7.689%); 8880/214836 objects misplaced (4.133%); 799 KiB/s, 12 objects/s recovering 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[113926]: osdmap e281: 8 total, 8 up, 7 in 2024-09-13T14:54:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16518/214836 objects degraded (7.689%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:16.393 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:16.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:16.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:16.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:16.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:16.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.730 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:16.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:16.831 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:16.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:16.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:16.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:16.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:17.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:17.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:17.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:17.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:17.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:17.387 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:17.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:17.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:17.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:17.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:17.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:17.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:17.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:17.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:17.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:17.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:17.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:17.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:18.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:18.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:18.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.185 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:18.185 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:18.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:18.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:18.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:17 smithi042 ceph-mon[113926]: pgmap v977: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16518/212478 objects degraded (7.774%); 8880/212478 objects misplaced (4.179%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:54:18.205 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:17 smithi042 ceph-mon[108570]: pgmap v977: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16518/212478 objects degraded (7.774%); 8880/212478 objects misplaced (4.179%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:54:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:17 smithi159 ceph-mon[102519]: pgmap v977: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16518/212478 objects degraded (7.774%); 8880/212478 objects misplaced (4.179%); 960 KiB/s, 14 objects/s recovering 2024-09-13T14:54:18.296 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:18.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:18.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:18.451 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:18.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:18.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:18.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:18.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:18.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:18.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:18.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:18.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:18.968 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:18.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T14:54:18.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:18.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:18.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:19.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:19.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:19.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:19.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:19.393 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:19.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:19.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:19.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:19.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:19.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:19] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:54:19.858 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:19.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:19.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:19.949 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:19.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:19.964 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T14:54:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.028 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:20.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:20.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:19 smithi159 ceph-mon[102519]: pgmap v978: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 16664/207822 objects degraded (8.018%); 8664/207822 objects misplaced (4.169%); 838 KiB/s, 13 objects/s recovering 2024-09-13T14:54:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1723642637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:19 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:20.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[113926]: pgmap v978: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 16664/207822 objects degraded (8.018%); 8664/207822 objects misplaced (4.169%); 838 KiB/s, 13 objects/s recovering 2024-09-13T14:54:20.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1723642637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[108570]: pgmap v978: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 16664/207822 objects degraded (8.018%); 8664/207822 objects misplaced (4.169%); 838 KiB/s, 13 objects/s recovering 2024-09-13T14:54:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1723642637' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:19 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T14:54:20.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:20.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:20.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:20.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:20.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:20.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:20.722 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:20.758 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:20.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T14:54:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:20.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:20.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:20.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:20.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:20.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:20.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[108570]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:54:20.994 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16664/207822 objects degraded (8.018%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:20.995 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:20.995 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.138 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:20 smithi159 ceph-mon[102519]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:54:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16664/207822 objects degraded (8.018%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:21.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T14:54:21.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[113926]: osdmap e282: 8 total, 8 up, 7 in 2024-09-13T14:54:21.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16664/207822 objects degraded (8.018%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:21.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:21.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:21.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:21.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:21.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:21.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:21.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.746 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:21.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:21.872 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:21.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:21.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:21.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:21.997 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.007 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:54:22.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:21 smithi159 ceph-mon[102519]: pgmap v980: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 16664/205491 objects degraded (8.109%); 8468/205491 objects misplaced (4.121%); 560 KiB/s, 8 objects/s recovering 2024-09-13T14:54:22.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:22.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:21 smithi042 ceph-mon[108570]: pgmap v980: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 16664/205491 objects degraded (8.109%); 8468/205491 objects misplaced (4.121%); 560 KiB/s, 8 objects/s recovering 2024-09-13T14:54:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:21 smithi042 ceph-mon[113926]: pgmap v980: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 16664/205491 objects degraded (8.109%); 8468/205491 objects misplaced (4.121%); 560 KiB/s, 8 objects/s recovering 2024-09-13T14:54:22.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:22.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:22.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:22.542 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:22.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:22.644 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:22.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.666 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:22.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:22.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:22.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:22.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:22.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:22.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:23.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:23.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:23.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:23.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:23.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:23.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:23.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:23.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:23.483 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:23.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:23.585 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:23.613 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:23.715 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:23.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:23.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:23.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:23.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:23.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:23.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:23.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:24.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:24.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:24.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:24.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:23 smithi159 ceph-mon[102519]: pgmap v981: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 16790/199098 objects degraded (8.433%); 8271/199098 objects misplaced (4.154%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:54:24.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:24.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:24.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:24 smithi042 ceph-mon[108570]: pgmap v981: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 16790/199098 objects degraded (8.433%); 8271/199098 objects misplaced (4.154%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:54:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:23 smithi042 ceph-mon[113926]: pgmap v981: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 16790/199098 objects degraded (8.433%); 8271/199098 objects misplaced (4.154%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T14:54:24.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:24.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:24.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:24.726 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:24.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:24.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:24.857 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:24.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:24.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:24.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:25.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.110 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:25.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:25.249 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:25.374 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:25.520 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:25.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:25.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:25.687 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:25.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:25.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:25.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:25.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:25.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:25.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:26.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:26.067 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:26.148 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:26.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:26 smithi042 ceph-mon[108570]: pgmap v982: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16790/196341 objects degraded (8.551%); 8091/196341 objects misplaced (4.121%); 948 KiB/s, 14 objects/s recovering 2024-09-13T14:54:26.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16790/196341 objects degraded (8.551%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:26.208 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:26 smithi042 ceph-mon[113926]: pgmap v982: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16790/196341 objects degraded (8.551%); 8091/196341 objects misplaced (4.121%); 948 KiB/s, 14 objects/s recovering 2024-09-13T14:54:26.209 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16790/196341 objects degraded (8.551%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:26.250 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:26.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:26.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:26.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:26.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.415 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:26.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:26 smithi159 ceph-mon[102519]: pgmap v982: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 16790/196341 objects degraded (8.551%); 8091/196341 objects misplaced (4.121%); 948 KiB/s, 14 objects/s recovering 2024-09-13T14:54:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16790/196341 objects degraded (8.551%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:26.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:26.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:26.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:26.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:26.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:26.780 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:26.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:26.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:26.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:26.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:26.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:26.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:27.072 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:27.115 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:27.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:27.240 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:27.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:27.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:27.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:27.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:27.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:27.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:27.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:27.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:27.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:27.741 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:27.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:27.865 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:27.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:27.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:27.993 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:28.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:28.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:28 smithi042 ceph-mon[108570]: pgmap v983: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16790/194061 objects degraded (8.652%); 8091/194061 objects misplaced (4.169%); 902 KiB/s, 14 objects/s recovering 2024-09-13T14:54:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1386350849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:28 smithi042 ceph-mon[113926]: pgmap v983: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16790/194061 objects degraded (8.652%); 8091/194061 objects misplaced (4.169%); 902 KiB/s, 14 objects/s recovering 2024-09-13T14:54:28.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1386350849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:28.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:28.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:28.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:28.260 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:28.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:28.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:28.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:28.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:28 smithi159 ceph-mon[102519]: pgmap v983: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16790/194061 objects degraded (8.652%); 8091/194061 objects misplaced (4.169%); 902 KiB/s, 14 objects/s recovering 2024-09-13T14:54:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1386350849' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:28.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:28.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:28.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:28.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:28.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:28.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:28.800 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:28.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:28.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:28.948 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:28.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:28.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:29.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:29.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:54:29.095 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:29.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:29.177 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.212 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:29.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:29.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:29.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:29.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:29.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:29.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:29.603 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:29.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:29.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:29.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:29.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:29.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:29.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:29.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:29.982 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:29] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.51.0" 2024-09-13T14:54:29.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:30.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:30.149 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:30.289 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:54:30.289 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:54:30.289 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:54:30.290 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:54:30.290 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:54:30.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:30 smithi042 ceph-mon[108570]: pgmap v984: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16913/189540 objects degraded (8.923%); 7880/189540 objects misplaced (4.157%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:54:30.290 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:30 smithi042 ceph-mon[113926]: pgmap v984: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16913/189540 objects degraded (8.923%); 7880/189540 objects misplaced (4.157%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:54:30.313 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:30.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:30.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:30 smithi159 ceph-mon[102519]: pgmap v984: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16913/189540 objects degraded (8.923%); 7880/189540 objects misplaced (4.157%); 1017 KiB/s, 15 objects/s recovering 2024-09-13T14:54:30.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:30.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:30.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:30.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:30.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:30.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:30.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:30.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:31.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:31.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.200 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:31.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:31.300 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16913/189540 objects degraded (8.923%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:31.300 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16913/189540 objects degraded (8.923%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:31.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:31.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:31.447 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:31.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:31.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16913/189540 objects degraded (8.923%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:31.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:31.593 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:31.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:31.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:31.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:31.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:31.820 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:31.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:31.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:31.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:31.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:32.039 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T14:54:32.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:32.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:32.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:32.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:32.265 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T14:54:32.312 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[113926]: pgmap v985: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 16913/187161 objects degraded (9.037%); 7689/187161 objects misplaced (4.108%); 970 KiB/s, 15 objects/s recovering 2024-09-13T14:54:32.312 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2107848600' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.312 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[108570]: pgmap v985: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 16913/187161 objects degraded (9.037%); 7689/187161 objects misplaced (4.108%); 970 KiB/s, 15 objects/s recovering 2024-09-13T14:54:32.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2107848600' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.313 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:32 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:32.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:32.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:32 smithi159 ceph-mon[102519]: pgmap v985: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 582 op/s; 16913/187161 objects degraded (9.037%); 7689/187161 objects misplaced (4.108%); 970 KiB/s, 15 objects/s recovering 2024-09-13T14:54:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2107848600' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:32 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T14:54:32.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:32.551 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:32.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:32.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:32.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:32.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:32.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:32.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:32.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:32.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:32.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:32.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:33.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.085 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:33.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.110 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:54:33.187 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.249 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:33.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:33 smithi042 ceph-mon[108570]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:54:33.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:33.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:33 smithi042 ceph-mon[113926]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:54:33.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:33.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:33.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T14:54:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:33 smithi159 ceph-mon[102519]: osdmap e283: 8 total, 8 up, 7 in 2024-09-13T14:54:33.520 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:33.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:33.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:33.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:33.769 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:33.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:33.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:33.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:33.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:33.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:33.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:34.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:34.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:34.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.145 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:34.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:34.272 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:34 smithi042 ceph-mon[113926]: pgmap v987: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 17022/180612 objects degraded (9.425%); 7498/180612 objects misplaced (4.151%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:34 smithi042 ceph-mon[108570]: pgmap v987: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 17022/180612 objects degraded (9.425%); 7498/180612 objects misplaced (4.151%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:34.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:34.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:34 smithi159 ceph-mon[102519]: pgmap v987: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 17022/180612 objects degraded (9.425%); 7498/180612 objects misplaced (4.151%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:34.500 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:34.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:34.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:34.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:34.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:34.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:34.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:34.893 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:34.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:34.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:35.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:35.084 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:35.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:35.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:35.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:35.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:35.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:35.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:35.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:35.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:35.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T14:54:35.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:35.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:35.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:35.870 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:35.883 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:35.896 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:35.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:35.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:36.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:36.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[108570]: pgmap v988: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 17022/177432 objects degraded (9.594%); 7284/177432 objects misplaced (4.105%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17022/177432 objects degraded (9.594%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[113926]: pgmap v988: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 17022/177432 objects degraded (9.594%); 7284/177432 objects misplaced (4.105%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17022/177432 objects degraded (9.594%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:36.417 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:36 smithi159 ceph-mon[102519]: pgmap v988: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 17022/177432 objects degraded (9.594%); 7284/177432 objects misplaced (4.105%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:36 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17022/177432 objects degraded (9.594%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:36.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:36.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:36.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:36.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:36.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:36.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:36.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:36.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:36.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:36.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:36.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:37.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.080 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:54:37.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:37.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:37.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:37.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:37.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.367 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:37.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:37.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:37.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:54:37.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:37.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:37.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:37.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:37.811 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:37.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:37.969 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:37.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:37.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:38.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:38.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:38.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:38.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:38.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[113926]: pgmap v989: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 17022/174984 objects degraded (9.728%); 7284/174984 objects misplaced (4.163%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:38.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:38.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:38.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[113926]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:54:38.256 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[108570]: pgmap v989: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 17022/174984 objects degraded (9.728%); 7284/174984 objects misplaced (4.163%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:38.256 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:38.256 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:38.256 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:38 smithi042 ceph-mon[108570]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:54:38.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:38.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:38.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:38.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:38.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:38 smithi159 ceph-mon[102519]: pgmap v989: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 17022/174984 objects degraded (9.728%); 7284/174984 objects misplaced (4.163%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T14:54:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:38 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:38 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3718940352' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:54:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:38 smithi159 ceph-mon[102519]: osdmap e284: 8 total, 8 up, 7 in 2024-09-13T14:54:38.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:38.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:38.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:38.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:38.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:38.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:38.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:38.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:38.928 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:39.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:39.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:39.343 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:39.380 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:39.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:39.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:39.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:39.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:39.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:39.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:39.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:39.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:54:39.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:39.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:39.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.020 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:39] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:54:40.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:40.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:40.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:40.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:40.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:40.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:40 smithi042 ceph-mon[113926]: pgmap v991: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 17146/170019 objects degraded (10.085%); 7087/170019 objects misplaced (4.168%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:40 smithi042 ceph-mon[108570]: pgmap v991: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 17146/170019 objects degraded (10.085%); 7087/170019 objects misplaced (4.168%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:40.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:40.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:40.462 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:40.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:40.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:40 smithi159 ceph-mon[102519]: pgmap v991: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.5 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 17146/170019 objects degraded (10.085%); 7087/170019 objects misplaced (4.168%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:54:40.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:40.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:40.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:40.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:40.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:40.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:40.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:40.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:40.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:41.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:41.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:41.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:41.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:41.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:41.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:41.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:41.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17146/170019 objects degraded (10.085%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:41.456 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17146/170019 objects degraded (10.085%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17146/170019 objects degraded (10.085%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:41.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:41.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:41.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:41.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:41.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:41.891 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:41.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:41.993 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:42.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118880 10.18 10:1b793137:::benchmark_data_smithi042_191918_object118879:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.84596743100002, 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48544058400000001, 2024-09-13T14:54:42.115 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118880 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264755+0000", 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264756+0000", 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4000000000000001e-07 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264778+0000", 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1586e-05 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264786+0000", 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0199999999999994e-06 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.264791+0000", 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6300000000000003e-06 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268136+0000", 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033446109999999999 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268289+0000", 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015365800000000001 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 5,7", 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268595+0000", 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000305569 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.371699+0000", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.103103899 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750047+0000", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.37834784300000002 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750062+0000", 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5671e-05 2024-09-13T14:54:42.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750144+0000", 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2013000000000002e-05 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750196+0000", 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1443999999999997e-05 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118882 10.5 10:a1c362d7:::benchmark_data_smithi042_191918_object118881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.84355429700003, 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48747538299999998, 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.123 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118882 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267168+0000", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267170+0000", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9609999999999999e-06 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267973+0000", 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00080228000000000005 2024-09-13T14:54:42.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267986+0000", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.296e-05 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.267993+0000", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4150000000000002e-06 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268200+0000", 2024-09-13T14:54:42.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000207099 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268293+0000", 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2695999999999995e-05 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.268535+0000", 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024242600000000001 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369281+0000", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.100745978 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369337+0000", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5943999999999998e-05 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754471+0000", 2024-09-13T14:54:42.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.38513415699999998 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754583+0000", 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000111717 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754644+0000", 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0749999999999999e-05 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.129 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118887 10.5 10:a472458b:::benchmark_data_smithi042_191918_object118886:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e210)", 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.83955751799999, 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.48549398199999999, 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.130 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118887 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271165+0000", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271167+0000", 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6899999999999999e-06 2024-09-13T14:54:42.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.271192+0000", 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4703999999999999e-05 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367626+0000", 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.096434612000000003 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.367639+0000", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2571e-05 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.369370+0000", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0017312440000000001 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.372032+0000", 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026619460000000001 2024-09-13T14:54:42.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749773+0000", 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.37774075400000001 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754657+0000", 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048840469999999999 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.754732+0000", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5648000000000005e-05 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756520+0000", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001787468 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756617+0000", 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6817000000000005e-05 2024-09-13T14:54:42.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.756659+0000", 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2481000000000002e-05 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118600 10.13 10:c8f84504:::benchmark_data_smithi042_191918_object118599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.36103159200002, 2024-09-13T14:54:42.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1358033599999999, 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118600 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749691+0000", 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749693+0000", 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.336e-06 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749768+0000", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5859000000000007e-05 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749780+0000", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1355000000000001e-05 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749787+0000", 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1740000000000003e-06 2024-09-13T14:54:42.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753090+0000", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003302868 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753098+0000", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5890000000000003e-06 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.875935+0000", 2024-09-13T14:54:42.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1228367180000001 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.876100+0000", 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016534899999999999 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.876294+0000", 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019339699999999999 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883004+0000", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067103090000000002 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883054+0000", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0083999999999997e-05 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883069+0000", 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4635e-05 2024-09-13T14:54:42.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.883129+0000", 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9762000000000001e-05 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885495+0000", 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023659250000000001 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118676 10.13 10:ca797c6a:::benchmark_data_smithi042_191918_object118675:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.36092436799998, 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.142041785, 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118676 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.146 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749798+0000", 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749799+0000", 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3399999999999998e-07 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749869+0000", 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9334999999999999e-05 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749878+0000", 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5810000000000003e-06 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749892+0000", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3489000000000001e-05 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753120+0000", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032279800000000001 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753126+0000", 2024-09-13T14:54:42.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9490000000000001e-06 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.880738+0000", 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1276121480000001 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.880910+0000", 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017218800000000001 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.881100+0000", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000189607 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885647+0000", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004547316 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885681+0000", 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3883000000000001e-05 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.889582+0000", 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039010870000000001 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.889640+0000", 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8053999999999997e-05 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.891840+0000", 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002200334 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118830 10.13 10:cb02c879:::benchmark_data_smithi042_191918_object118829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.36081667399998, 2024-09-13T14:54:42.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1522871590000001, 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118830 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:54:42.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749906+0000", 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749907+0000", 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6499999999999999e-07 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749960+0000", 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2935000000000001e-05 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749968+0000", 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5459999999999992e-06 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749974+0000", 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1630000000000001e-06 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753147+0000", 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031729150000000001 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753153+0000", 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0689999999999998e-06 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885720+0000", 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1325662890000001 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.885877+0000", 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015730400000000001 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.886036+0000", 2024-09-13T14:54:42.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015868800000000001 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899913+0000", 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013877633 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899985+0000", 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1768999999999997e-05 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900133+0000", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000147818 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900205+0000", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2096999999999998e-05 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902193+0000", 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019882680000000001 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.160 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118868 10.13 10:cd95e6bd:::benchmark_data_smithi042_191918_object118867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.36073749000002, 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.154318097, 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.161 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118868 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749985+0000", 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.749986+0000", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7800000000000001e-07 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750035+0000", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8702999999999998e-05 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750042+0000", 2024-09-13T14:54:42.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3869999999999997e-06 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.750048+0000", 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6810000000000003e-06 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753173+0000", 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031257099999999999 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.753179+0000", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8150000000000002e-06 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887424+0000", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1342447490000001 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887532+0000", 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108297 2024-09-13T14:54:42.165 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.887710+0000", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017781200000000001 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.899925+0000", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012215014 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900056+0000", 2024-09-13T14:54:42.166 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013064500000000001 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902264+0000", 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022082170000000002 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.902331+0000", 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6826e-05 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904303+0000", 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019726629999999999 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:118904 10.13 10:cfdaed63:::benchmark_data_smithi042_191918_object118903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.35567276799998, 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.155650617, 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.168 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 118904 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.169 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755050+0000", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755052+0000", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6109999999999999e-06 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755096+0000", 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4175000000000003e-05 2024-09-13T14:54:42.170 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755109+0000", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3227999999999999e-05 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755117+0000", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3010000000000007e-06 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755168+0000", 2024-09-13T14:54:42.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1125999999999998e-05 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.755175+0000", 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5699999999999998e-06 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.891904+0000", 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.13672928 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.172 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.892011+0000", 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107039 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.892121+0000", 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109954 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900067+0000", 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079452470000000008 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.173 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908305+0000", 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082383319999999993 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908429+0000", 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012412400000000001 2024-09-13T14:54:42.174 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908511+0000", 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1971000000000005e-05 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.910701+0000", 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002189659 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.175 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.177 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119047 10.13 10:cb7567d0:::benchmark_data_smithi042_191918_object119046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.210123259, 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0174962750000001, 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119047 2024-09-13T14:54:42.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900600+0000", 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900601+0000", 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7540000000000001e-06 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900641+0000", 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9452000000000001e-05 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900660+0000", 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9264000000000001e-05 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900670+0000", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0199e-05 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900809+0000", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013844000000000001 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.900820+0000", 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1568e-05 2024-09-13T14:54:42.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.893864+0000", 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99304427299999998 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.894029+0000", 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016483599999999999 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.182 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.894225+0000", 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000196181 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900071+0000", 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058458820000000002 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915505+0000", 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015433293000000001 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.183 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915663+0000", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015855399999999999 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915735+0000", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1389000000000004e-05 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918096+0000", 2024-09-13T14:54:42.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00236119 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119065 10.13 10:ced35f11:::benchmark_data_smithi042_191918_object119064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.19552692399998, 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.005246254, 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.185 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119065 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915196+0000", 2024-09-13T14:54:42.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915198+0000", 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7370000000000001e-06 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915220+0000", 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2031999999999998e-05 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915235+0000", 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4924e-05 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915245+0000", 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0465000000000001e-05 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915303+0000", 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7815000000000001e-05 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.915311+0000", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5930000000000006e-06 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.895795+0000", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98048344899999995 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.895961+0000", 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016654500000000001 2024-09-13T14:54:42.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.896165+0000", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020370800000000001 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.900076+0000", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039105750000000003 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915558+0000", 2024-09-13T14:54:42.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015482490999999999 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918158+0000", 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025996370000000001 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.918283+0000", 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012484900000000001 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920442+0000", 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002159434 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119099 10.13 10:ce67b412:::benchmark_data_smithi042_191918_object119098:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.163456999, 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98944306000000004, 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.192 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119099 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.193 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947266+0000", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947268+0000", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8419999999999999e-06 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947297+0000", 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.915e-05 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947312+0000", 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4877e-05 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947320+0000", 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2770000000000005e-06 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947390+0000", 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9997000000000001e-05 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.947401+0000", 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0635999999999999e-05 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.897811+0000", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.95041027600000005 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.897976+0000", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000165241 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.898151+0000", 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017505900000000001 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904317+0000", 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061655779999999997 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915606+0000", 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011289334 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.934323+0000", 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018716602999999998 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.934382+0000", 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9710000000000003e-05 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936709+0000", 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023264800000000001 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.199 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119117 10.13 10:cfc8a476:::benchmark_data_smithi042_191918_object119116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.14979316400002, 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97974125099999998, 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119117 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960930+0000", 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960931+0000", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-06 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960951+0000", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0151000000000001e-05 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960962+0000", 2024-09-13T14:54:42.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0699e-05 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.960967+0000", 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8810000000000004e-06 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.961022+0000", 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5495000000000003e-05 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.961031+0000", 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.884e-06 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904350+0000", 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94331945500000003 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904516+0000", 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016536900000000001 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.904724+0000", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000208402 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.908320+0000", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035956809999999999 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922915+0000", 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014594817 2024-09-13T14:54:42.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936800+0000", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013885085 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936842+0000", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2058000000000003e-05 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940671+0000", 2024-09-13T14:54:42.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038289629999999999 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119145 10.13 10:ce15cb28:::benchmark_data_smithi042_191918_object119144:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.11607526099999, 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94818519499999998, 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.207 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119145 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994648+0000", 2024-09-13T14:54:42.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994649+0000", 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3260000000000001e-06 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994668+0000", 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9227000000000001e-05 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994679+0000", 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0873000000000001e-05 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994686+0000", 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2529999999999998e-06 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994771+0000", 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5223999999999996e-05 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:15.994783+0000", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.204e-05 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906355+0000", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.91157189199999999 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906521+0000", 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016598599999999999 2024-09-13T14:54:42.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.906723+0000", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020175600000000001 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.915445+0000", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0087218490000000003 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922989+0000", 2024-09-13T14:54:42.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075438839999999998 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940736+0000", 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017747338000000001 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.940803+0000", 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6346000000000005e-05 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942833+0000", 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020302010000000001 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119172 10.13 10:c92e9377:::benchmark_data_smithi042_191918_object119171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.08872484199998, 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92329959699999997, 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.214 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119172 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.021998+0000", 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022006+0000", 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4209999999999995e-06 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022041+0000", 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4915e-05 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022053+0000", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.204e-05 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022069+0000", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5824999999999998e-05 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022126+0000", 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7309000000000003e-05 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.022135+0000", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5490000000000006e-06 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913097+0000", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.89096235999999995 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913266+0000", 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000168927 2024-09-13T14:54:42.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.913463+0000", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019709 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.922930+0000", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.009466805 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923085+0000", 2024-09-13T14:54:42.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015497500000000001 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942911+0000", 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019825559 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.942984+0000", 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3442000000000006e-05 2024-09-13T14:54:42.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945298+0000", 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023133799999999999 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119218 10.13 10:cf0f2136:::benchmark_data_smithi042_191918_object119217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.04104604000003, 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.87838980200000005, 2024-09-13T14:54:42.221 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119218 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069677+0000", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0400000000000002e-07 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069694+0000", 2024-09-13T14:54:42.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6555e-05 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069699+0000", 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6269999999999997e-06 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069703+0000", 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7000000000000002e-06 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069763+0000", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9670999999999998e-05 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.069769+0000", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.993e-06 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920503+0000", 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.850734412 2024-09-13T14:54:42.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920653+0000", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015012900000000001 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.920809+0000", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000156012 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.931881+0000", 2024-09-13T14:54:42.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.01107209 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945353+0000", 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013471854 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945401+0000", 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7459000000000001e-05 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.945460+0000", 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8893e-05 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.948067+0000", 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026070030000000001 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119242 10.13 10:ced1767e:::benchmark_data_smithi042_191918_object119241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.228 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.01099261799999, 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.85350327299999995, 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119242 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099730+0000", 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099732+0000", 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4020000000000001e-06 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099757+0000", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5341e-05 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099776+0000", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9567000000000001e-05 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099785+0000", 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7600000000000008e-06 2024-09-13T14:54:42.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099874+0000", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9054999999999994e-05 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099885+0000", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0862e-05 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923144+0000", 2024-09-13T14:54:42.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.82325866999999997 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923298+0000", 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153821 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.923484+0000", 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000186331 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936730+0000", 2024-09-13T14:54:42.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:42.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013246002999999999 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950271+0000", 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013541169 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950313+0000", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1520000000000002e-05 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950370+0000", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7769000000000002e-05 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953233+0000", 2024-09-13T14:54:42.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028630029999999998 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119243 10.13 10:cc7c2522:::benchmark_data_smithi042_191918_object119242:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.01082089200003, 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.85619771600000005, 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.237 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119243 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099902+0000", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099903+0000", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7000000000000003e-07 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099927+0000", 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4298999999999999e-05 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099941+0000", 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4433999999999999e-05 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.099949+0000", 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7610000000000007e-06 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.100020+0000", 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0569999999999997e-05 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.100031+0000", 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1406999999999999e-05 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925483+0000", 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.82545171500000003 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925631+0000", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014806199999999999 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.925810+0000", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017913599999999999 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936738+0000", 2024-09-13T14:54:42.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010927377 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953455+0000", 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016717868 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953567+0000", 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011176299999999999 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953622+0000", 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4991999999999998e-05 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.956100+0000", 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024774620000000002 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119249 10.13 10:cbddd720:::benchmark_data_smithi042_191918_object119248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 386.00172026899997, 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.86949032100000001, 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119249 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109003+0000", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109005+0000", 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1289999999999999e-06 2024-09-13T14:54:42.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109031+0000", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6843999999999999e-05 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109054+0000", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2503000000000001e-05 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109065+0000", 2024-09-13T14:54:42.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0645000000000001e-05 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109125+0000", 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0517999999999999e-05 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.109132+0000", 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1879999999999996e-06 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.927711+0000", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.81857863099999995 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.927847+0000", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013632299999999999 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.928012+0000", 2024-09-13T14:54:42.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164556 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.936745+0000", 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0087328030000000008 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.953515+0000", 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016770804 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.250 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.976014+0000", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022498335000000001 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.976077+0000", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3629999999999999e-05 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978493+0000", 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002415412 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.251 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:119304 10.13 10:cb790c47:::benchmark_data_smithi042_191918_object119303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 385.78972790799997, 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.65975735199999996, 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.252 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 119304 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320995+0000", 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.320996+0000", 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4419999999999999e-06 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321018+0000", 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1437000000000002e-05 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.254 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321032+0000", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3841e-05 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321039+0000", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.126e-06 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321095+0000", 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.596e-05 2024-09-13T14:54:42.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.321103+0000", 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6600000000000001e-06 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.929709+0000", 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.60860574199999995 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.929886+0000", 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017648699999999999 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.930067+0000", 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000180951 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.950222+0000", 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020155395 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.958813+0000", 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0085912220000000008 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978545+0000", 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019732178 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.978614+0000", 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9048999999999996e-05 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:16.980752+0000", 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021378619999999999 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45314.0:135201 11.d 11:ba664167:::benchmark_data_smithi042_223320_object16195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e274)", 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:53:23.673338+0000", 2024-09-13T14:54:42.259 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 78.437384412, 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.42382509600000001, 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45314", 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 135201 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673338+0000", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673338+0000", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673340+0000", 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.511e-06 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.261 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673345+0000", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2569999999999998e-06 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673358+0000", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2809000000000001e-05 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.673367+0000", 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5879999999999998e-06 2024-09-13T14:54:42.262 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.699787+0000", 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.026420381999999999 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for rw locks", 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:23.700014+0000", 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022745900000000001 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:54:42.263 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.049728+0000", 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.34971327299999999 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.049981+0000", 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025359900000000002 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.050400+0000", 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041871200000000001 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.264 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.075482+0000", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.025082184 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.075536+0000", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3378999999999998e-05 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.091923+0000", 2024-09-13T14:54:42.265 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016387632999999999 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.092077+0000", 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153435 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:53:24.097163+0000", 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0050868750000000003 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:54:42.266 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:54:42.267 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:54:42.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:42 smithi042 ceph-mon[108570]: pgmap v992: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17146/167403 objects degraded (10.242%); 6870/167403 objects misplaced (4.104%); 555 KiB/s, 8 objects/s recovering 2024-09-13T14:54:42.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:42 smithi042 ceph-mon[113926]: pgmap v992: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17146/167403 objects degraded (10.242%); 6870/167403 objects misplaced (4.104%); 555 KiB/s, 8 objects/s recovering 2024-09-13T14:54:42.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:42.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:42.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:42 smithi159 ceph-mon[102519]: pgmap v992: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17146/167403 objects degraded (10.242%); 6870/167403 objects misplaced (4.104%); 555 KiB/s, 8 objects/s recovering 2024-09-13T14:54:42.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:42.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:42.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:42.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:42.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:42.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:42.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:42.876 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:42.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:42.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:42.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:43.020 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.062 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.304 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:43.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:43.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:43.430 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:43.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:43.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:43.598 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:43.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:43.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:43.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:43.893 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:43.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:43.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:43.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:44.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.120 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:44.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:44.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:44.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:44.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:44 smithi159 ceph-mon[102519]: pgmap v993: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/160887 objects degraded (10.726%); 6685/160887 objects misplaced (4.155%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:44.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:44.542 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:44.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:44 smithi042 ceph-mon[113926]: pgmap v993: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/160887 objects degraded (10.726%); 6685/160887 objects misplaced (4.155%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:44.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:44 smithi042 ceph-mon[108570]: pgmap v993: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/160887 objects degraded (10.726%); 6685/160887 objects misplaced (4.155%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:44.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:44.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:44.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:44.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:44.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:44.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:44.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:44.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:44.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.079 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:45.120 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:45.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:45.222 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:45.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.281 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:45.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:45.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:45.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:45.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:45.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:45.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:45.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:45.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:45.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:45.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:45.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:45.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:45.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:46.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.124 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:54:46.124 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:54:46.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:46.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:54:46.172 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:54:46.219 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:54:46.219 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:54:46.232 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.237 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:46.281 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.289 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:54:46.290 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:54:46.329 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:54:46.330 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:54:46.368 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:54:46.369 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:54:46.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[108570]: pgmap v994: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/157710 objects degraded (10.942%); 6511/157710 objects misplaced (4.128%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:46.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17257/157710 objects degraded (10.942%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:46.369 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2713743672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:46.370 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[113926]: pgmap v994: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/157710 objects degraded (10.942%); 6511/157710 objects misplaced (4.128%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:46.370 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17257/157710 objects degraded (10.942%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:46.370 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:46 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2713743672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:46.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:46.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:46.418 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:54:46.418 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:54:46.452 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:54:46.453 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:54:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:46 smithi159 ceph-mon[102519]: pgmap v994: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 17257/157710 objects degraded (10.942%); 6511/157710 objects misplaced (4.128%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:46 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17257/157710 objects degraded (10.942%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:46 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2713743672' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:54:46.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:46.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:46.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.711 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:46.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:46.812 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:46.850 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:46.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:46.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:46.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:54:46.952 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.013 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:47.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:47.138 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:47.146 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1541673262' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.146 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:47 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1541673262' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:47 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.147 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T14:54:47.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:47.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:47.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:47.366 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:47.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:47.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:47.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1541673262' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:47 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T14:54:47.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:47.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:47.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:47.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:47.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.809 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:47.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:47.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:47.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:47.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:47.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:48.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:48.083 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:48.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T14:54:48.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:48.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:48.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:48.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:48.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:48.416 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[113926]: pgmap v995: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 17257/155346 objects degraded (11.109%); 6511/155346 objects misplaced (4.191%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:48.416 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:48.416 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[113926]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:54:48.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[108570]: pgmap v995: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 17257/155346 objects degraded (11.109%); 6511/155346 objects misplaced (4.191%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:48.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:48.418 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:48 smithi042 ceph-mon[108570]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:54:48.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:48.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:48.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:48.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:48 smithi159 ceph-mon[102519]: pgmap v995: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 17257/155346 objects degraded (11.109%); 6511/155346 objects misplaced (4.191%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:48 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T14:54:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:48 smithi159 ceph-mon[102519]: osdmap e285: 8 total, 8 up, 7 in 2024-09-13T14:54:48.591 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:48.617 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:48.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:48.753 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:48.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:48.896 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:48.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:48.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:48.997 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:49.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:49.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:49.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:49.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:49.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:49.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:49.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:49.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:49.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:49.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:49.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:49.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:49.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:49.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:49.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:49.818 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:49.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:49.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:49.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:49.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:50.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:49] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:54:50.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:50.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:50.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:50.240 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:50.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:50.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:50.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:50.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:50.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:50 smithi159 ceph-mon[102519]: pgmap v997: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 17384/150615 objects degraded (11.542%); 6311/150615 objects misplaced (4.190%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:50.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:50.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:50 smithi042 ceph-mon[108570]: pgmap v997: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 17384/150615 objects degraded (11.542%); 6311/150615 objects misplaced (4.190%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:50 smithi042 ceph-mon[113926]: pgmap v997: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 17384/150615 objects degraded (11.542%); 6311/150615 objects misplaced (4.190%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:50.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:50.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:50.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.780 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:50.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:50.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:50.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:50.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:50.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:51.032 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:51.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:51.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:51.284 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:51.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:51.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:51.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:51 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17384/150615 objects degraded (11.542%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:51.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:51.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:51.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:51.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17384/150615 objects degraded (11.542%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:51.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:51.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:51.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17384/150615 objects degraded (11.542%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:51.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:54:51.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:54:51.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:51.694 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:51.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:51.819 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:51.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:51.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:51.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:52.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:52.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:52.151 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:52.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.248 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:52 smithi042 ceph-mon[108570]: pgmap v998: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 17384/148185 objects degraded (11.731%); 6102/148185 objects misplaced (4.118%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:52 smithi042 ceph-mon[113926]: pgmap v998: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 17384/148185 objects degraded (11.731%); 6102/148185 objects misplaced (4.118%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:52.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:52.397 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:52.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:52 smithi159 ceph-mon[102519]: pgmap v998: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 17384/148185 objects degraded (11.731%); 6102/148185 objects misplaced (4.118%); 940 KiB/s, 14 objects/s recovering 2024-09-13T14:54:52.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:52.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:52.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:52.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:52.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:52.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:52.798 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:52.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:52.950 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:52.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:53.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:53.102 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:53.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:53.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:53.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:53.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:53.362 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:53.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:53.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:53.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:53.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:53.630 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:53.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:53.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:54:53.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:53.755 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:53.857 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:53.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:53.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:53.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:54.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:54.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:54.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:54.178 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:54:54.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:54.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:54.382 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:54.404 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:54 smithi159 ceph-mon[102519]: pgmap v999: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17501/141342 objects degraded (12.382%); 5906/141342 objects misplaced (4.179%); 972 KiB/s, 15 objects/s recovering 2024-09-13T14:54:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1486685762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:54 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:54.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:54.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[113926]: pgmap v999: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17501/141342 objects degraded (12.382%); 5906/141342 objects misplaced (4.179%); 972 KiB/s, 15 objects/s recovering 2024-09-13T14:54:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1486685762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[108570]: pgmap v999: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 17501/141342 objects degraded (12.382%); 5906/141342 objects misplaced (4.179%); 972 KiB/s, 15 objects/s recovering 2024-09-13T14:54:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1486685762' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:54 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:54:54.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:54.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:54.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:54.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:54.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:54.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:54.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:54.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:54.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:54.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:55.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:55.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.078 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:55.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:55.179 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:55.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:55.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:55.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:55.353 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:55.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:55.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:55.478 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:55.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:55 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:55 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:55 smithi159 ceph-mon[102519]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[113926]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:54:55.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:55 smithi042 ceph-mon[108570]: osdmap e286: 8 total, 8 up, 7 in 2024-09-13T14:54:55.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:55.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:55.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:55.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:55.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:55.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:55.890 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:55.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:55.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:55.990 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:54:55.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:56.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:56.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:56.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.164 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:56.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:56.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:56.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:56.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:56.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:56 smithi042 ceph-mon[108570]: pgmap v1001: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 17501/138090 objects degraded (12.674%); 5738/138090 objects misplaced (4.155%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:54:56.296 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17501/138090 objects degraded (12.674%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:56.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:56 smithi042 ceph-mon[113926]: pgmap v1001: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 17501/138090 objects degraded (12.674%); 5738/138090 objects misplaced (4.155%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:54:56.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17501/138090 objects degraded (12.674%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:56.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:56.433 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:56.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:56 smithi159 ceph-mon[102519]: pgmap v1001: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 718 op/s; 17501/138090 objects degraded (12.674%); 5738/138090 objects misplaced (4.155%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T14:54:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17501/138090 objects degraded (12.674%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:54:56.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:56.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:56.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:54:56.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:54:56.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:56.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.872 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:56.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:56.876 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:54:56.876 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:54:56.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:56.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:56.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:56.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:57.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:57.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:57.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:57.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.412 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:57.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:57.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:57.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:57.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:54:57.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:57.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:57.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:54:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:57.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:54:57.923 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:57.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:57.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:57.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:58.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:58.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.214 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T14:54:58.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:58.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:58.361 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.463 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:58.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:58.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:58.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:58 smithi159 ceph-mon[102519]: pgmap v1002: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 17501/135534 objects degraded (12.913%); 5738/135534 objects misplaced (4.234%); 543 KiB/s, 8 objects/s recovering 2024-09-13T14:54:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3144028211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:58 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[113926]: pgmap v1002: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 17501/135534 objects degraded (12.913%); 5738/135534 objects misplaced (4.234%); 543 KiB/s, 8 objects/s recovering 2024-09-13T14:54:58.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3144028211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[108570]: pgmap v1002: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 17501/135534 objects degraded (12.913%); 5738/135534 objects misplaced (4.234%); 543 KiB/s, 8 objects/s recovering 2024-09-13T14:54:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3144028211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:58 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T14:54:58.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:58.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:58.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:58.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:58.846 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:54:58.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:54:58.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:58.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:58.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:58.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:59.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:59.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:59.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:54:59.226 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:54:59 smithi042 ceph-mon[108570]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:54:59.249 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:59.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:59.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:54:59.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:54:59.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:54:59.497 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:54:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:54:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:54:59.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:54:59 smithi159 ceph-mon[102519]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:54:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T14:54:59.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:54:59 smithi042 ceph-mon[113926]: osdmap e287: 8 total, 8 up, 7 in 2024-09-13T14:54:59.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:54:59.621 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:59.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:54:59.723 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:54:59.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:59.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:54:59.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:54:59.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:54:59.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:54:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:54:59.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:54:59.970 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:54:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:54:59] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.51.0" 2024-09-13T14:54:59.980 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:00.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:00.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:00.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.125 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:00.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:00.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:00.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:00.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:00.378 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:00.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:00 smithi159 ceph-mon[102519]: pgmap v1004: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 735 op/s; 17640/130533 objects degraded (13.514%); 5520/130533 objects misplaced (4.229%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:00 smithi159 ceph-mon[102519]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:55:00.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:00 smithi042 ceph-mon[113926]: pgmap v1004: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 735 op/s; 17640/130533 objects degraded (13.514%); 5520/130533 objects misplaced (4.229%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:00.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:00 smithi042 ceph-mon[113926]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:55:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:00 smithi042 ceph-mon[108570]: pgmap v1004: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 735 op/s; 17640/130533 objects degraded (13.514%); 5520/130533 objects misplaced (4.229%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:00 smithi042 ceph-mon[108570]: osdmap e288: 8 total, 8 up, 7 in 2024-09-13T14:55:00.610 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:00.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:00.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:00.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:00.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:00.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:00.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:00.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:00.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:01.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:01.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:01.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:01.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:01.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[113926]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:55:01.265 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[113926]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 286'12251 2024-09-13T14:55:01.265 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17640/130533 objects degraded (13.514%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:01.265 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[113926]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:55:01.265 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/499101462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.268 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[108570]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:55:01.268 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[108570]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 286'12251 2024-09-13T14:55:01.268 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17640/130533 objects degraded (13.514%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:01.268 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[108570]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:55:01.269 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/499101462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.290 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:01.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:01.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:01 smithi159 ceph-mon[102519]: osdmap e289: 8 total, 8 up, 7 in 2024-09-13T14:55:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:01 smithi159 ceph-mon[102519]: 11.9 starting backfill to osd.0 from (0'0,0'0] MAX to 286'12251 2024-09-13T14:55:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:01 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17640/130533 objects degraded (13.514%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:01.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:01 smithi159 ceph-mon[102519]: osdmap e290: 8 total, 8 up, 7 in 2024-09-13T14:55:01.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/499101462' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:01.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:01.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:01.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:01.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:01.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:01.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:01.829 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:01.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:01.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:01.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:01.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:01.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:02.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:02.147 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:02.148 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:02.149 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:55:02.149 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:55:02.149 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:55:02.149 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:55:02.152 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:02.152 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:02.152 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:02.189 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:02.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:02.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:02.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.444 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:02 smithi159 ceph-mon[102519]: pgmap v1008: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 17640/128322 objects degraded (13.747%); 5341/128322 objects misplaced (4.162%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:02 smithi159 ceph-mon[102519]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:55:02.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:02.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:02.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:02 smithi042 ceph-mon[108570]: pgmap v1008: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 17640/128322 objects degraded (13.747%); 5341/128322 objects misplaced (4.162%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:02 smithi042 ceph-mon[108570]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:55:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:02 smithi042 ceph-mon[113926]: pgmap v1008: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 17640/128322 objects degraded (13.747%); 5341/128322 objects misplaced (4.162%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T14:55:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:02 smithi042 ceph-mon[113926]: osdmap e291: 8 total, 8 up, 7 in 2024-09-13T14:55:02.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:02.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:02.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:02.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:02.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:02.944 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:02.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:02.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:03.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:03.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:03.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:03.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:03.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:03.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:03.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:03.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:03.471 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:03.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:03.614 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:03.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:03.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:03.741 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:03.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:03.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:03.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:03.905 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T14:55:03.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:03.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:04.014 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:04.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:04.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:04.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:04.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:04.312 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:04.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:04.412 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:04.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:04.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:04 smithi159 ceph-mon[102519]: pgmap v1010: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 17657/124260 objects degraded (14.210%); 7734/124260 objects misplaced (6.224%); 2.8 MiB/s, 46 objects/s recovering 2024-09-13T14:55:04.539 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:04 smithi042 ceph-mon[108570]: pgmap v1010: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 17657/124260 objects degraded (14.210%); 7734/124260 objects misplaced (6.224%); 2.8 MiB/s, 46 objects/s recovering 2024-09-13T14:55:04.540 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:04 smithi042 ceph-mon[113926]: pgmap v1010: 99 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 17657/124260 objects degraded (14.210%); 7734/124260 objects misplaced (6.224%); 2.8 MiB/s, 46 objects/s recovering 2024-09-13T14:55:04.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:04.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:04.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:04.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:04.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:04.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:04.954 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:04.984 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:04.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:05.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:05.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:05.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:05.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.512 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:05.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:05.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:05.657 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:05.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:05.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:05.780 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:05.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:05.910 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:05.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:05.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:06.012 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:06.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:06.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:06.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:06.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:06.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:06.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:06.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.481 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:06.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:06 smithi159 ceph-mon[102519]: pgmap v1011: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 14566/122208 objects degraded (11.919%); 10030/122208 objects misplaced (8.207%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:06 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14566/122208 objects degraded (11.919%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:06.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[113926]: pgmap v1011: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 14566/122208 objects degraded (11.919%); 10030/122208 objects misplaced (8.207%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:06.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14566/122208 objects degraded (11.919%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[108570]: pgmap v1011: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 388 op/s; 14566/122208 objects degraded (11.919%); 10030/122208 objects misplaced (8.207%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14566/122208 objects degraded (11.919%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:06.603 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:06.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:06.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:06.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:06.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:06.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:06.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:06.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:06.980 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:07.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:07.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.172 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:07.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:07.307 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:07.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:07.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:07.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:07.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:07.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:07.606 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:07.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:07.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:07.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:07.860 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:07.870 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:07.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:07.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:07.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:08.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:08.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:08.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:08.225 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:08.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:08.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:08.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:08 smithi042 ceph-mon[108570]: pgmap v1012: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 14566/119811 objects degraded (12.157%); 9841/119811 objects misplaced (8.214%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:08 smithi042 ceph-mon[113926]: pgmap v1012: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 14566/119811 objects degraded (12.157%); 9841/119811 objects misplaced (8.214%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:08.351 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:08.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.375 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:08.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:08.476 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:08 smithi159 ceph-mon[102519]: pgmap v1012: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 14566/119811 objects degraded (12.157%); 9841/119811 objects misplaced (8.214%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:08.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:08.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:08.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:08.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:08.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:08.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:08.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:08.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:08.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:08.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:09.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:09.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:09.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:09.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:09.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.267 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:09 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:09.267 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:09 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:09.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:09.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:09.476 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:09.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.537 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:09.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:09.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:09.660 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:09 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:55:09.762 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:09.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:09.887 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:09.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:09.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:09.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:10.060 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:10.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:09] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-13T14:55:10.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:10.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:10.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:10.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:10.341 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.443 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:10.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:10.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:10.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:10.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:10 smithi042 ceph-mon[113926]: pgmap v1013: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14594/115137 objects degraded (12.675%); 9680/115137 objects misplaced (8.407%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:55:10.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:10 smithi042 ceph-mon[108570]: pgmap v1013: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14594/115137 objects degraded (12.675%); 9680/115137 objects misplaced (8.407%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:55:10.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:10.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:10.743 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:10.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:10 smithi159 ceph-mon[102519]: pgmap v1013: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 14594/115137 objects degraded (12.675%); 9680/115137 objects misplaced (8.407%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:55:10.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:10.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:10.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:10.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:10.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:11.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:11.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:11.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:11.140 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:11.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:11.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:11.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:11.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:11.413 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:11.515 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:11.543 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:11.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:11.594 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:11 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14594/115137 objects degraded (12.675%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:11.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:11 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14594/115137 objects degraded (12.675%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:11.640 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:11 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14594/115137 objects degraded (12.675%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:11.645 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:11.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:11.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:11.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:11.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:12.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:12.091 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:12.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:12.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:12.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:12 smithi042 ceph-mon[113926]: pgmap v1014: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14731/112665 objects degraded (13.075%); 9290/112665 objects misplaced (8.246%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:55:12.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:12 smithi042 ceph-mon[108570]: pgmap v1014: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14731/112665 objects degraded (13.075%); 9290/112665 objects misplaced (8.246%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:55:12.415 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:12.517 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:12.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:12.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:12.553 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:12 smithi159 ceph-mon[102519]: pgmap v1014: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 14731/112665 objects degraded (13.075%); 9290/112665 objects misplaced (8.246%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:55:12.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:12.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:12.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:12.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:12.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:12.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:12.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:12.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:12.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:13.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:13.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:13.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:13.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:13.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:13.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:13.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:13.384 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:13.404 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:13.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:13.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.551 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:13.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:13.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:13.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:13.782 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:13.803 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:55:13.805 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:55:13.821 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:13.923 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:13.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:13.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:13.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:14.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:14.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:14.213 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:14.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:14.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:14.287 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:55:14.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:14.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.395 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:14.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[113926]: pgmap v1015: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14802/106134 objects degraded (13.947%); 8892/106134 objects misplaced (8.378%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/698120837' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[108570]: pgmap v1015: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14802/106134 objects degraded (13.947%); 8892/106134 objects misplaced (8.378%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/698120837' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:14.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:14.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:14.551 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:14 smithi159 ceph-mon[102519]: pgmap v1015: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14802/106134 objects degraded (13.947%); 8892/106134 objects misplaced (8.378%); 3.5 MiB/s, 56 objects/s recovering 2024-09-13T14:55:14.551 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/698120837' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.551 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:55:14.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:14.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:14.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:14.795 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:14.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:14.946 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:14.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:14.962 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:55:14.962 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:55:14.962 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:55:14.962 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:55:14.962 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:55:15.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:15.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:15.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:15.200 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:15.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:15.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:15 smithi159 ceph-mon[102519]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:15 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:15 smithi159 ceph-mon[102519]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:55:15.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:15.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:15.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[108570]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[108570]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[113926]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:55:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:15 smithi042 ceph-mon[113926]: osdmap e292: 8 total, 8 up, 7 in 2024-09-13T14:55:15.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:15.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:15.834 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:15.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:15.895 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1015,"stamp":"2024-09-13T14:55:12.461923+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2317359101,"num_objects":35378,"num_object_clones":0,"num_object_copies":106134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14802,"num_objects_misplaced":8892,"num_objects_unfound":0,"num_objects_dirty":35378,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202720,"num_write_kb":7616226,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3402,"num_bytes_recovered":219682312,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111450,"ondisk_log_size":111450,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11395992,"kb_used_data":6292704,"kb_used_omap":27,"kb_used_meta":5103076,"kb_avail":644820072,"statfs":{"total":671965249536,"available":660295753728,"internally_reserved":0,"allocated":6443728896,"data_stored":6432633209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5225550795},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,10,5,3,10,4,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-453378048,"num_objects":-6918,"num_object_clones":0,"num_object_copies":-20754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2838,"num_objects_misplaced":1012,"num_objects_unfound":0,"num_objects_dirty":-6918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6918,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":623,"num_bytes_recovered":40500688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.998100"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":405,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.465874+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:55:01.465874+0000","last_peered":"2024-09-13T14:55:01.465874+0000","last_clean":"2024-09-13T14:55:01.465874+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:55:01.465874+0000","last_undegraded":"2024-09-13T14:55:01.465874+0000","last_fullsized":"2024-09-13T14:55:01.465874+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":71,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.461792+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:55:01.461792+0000","last_peered":"2024-09-13T14:55:01.461792+0000","last_clean":"2024-09-13T14:55:01.461792+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:55:01.461792+0000","last_undegraded":"2024-09-13T14:55:01.461792+0000","last_fullsized":"2024-09-13T14:55:01.461792+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":259,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.444018+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:55:00.444018+0000","last_peered":"2024-09-13T14:55:00.444018+0000","last_clean":"2024-09-13T14:55:00.444018+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:55:00.444018+0000","last_undegraded":"2024-09-13T14:55:00.444018+0000","last_fullsized":"2024-09-13T14:55:00.444018+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":97,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.471852+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:55:01.471852+0000","last_peered":"2024-09-13T14:55:01.471852+0000","last_clean":"2024-09-13T14:55:01.471852+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:55:01.471852+0000","last_undegraded":"2024-09-13T14:55:01.471852+0000","last_fullsized":"2024-09-13T14:55:01.471852+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":97,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478867+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:55:01.478867+0000","last_peered":"2024-09-13T14:55:01.478867+0000","last_clean":"2024-09-13T14:55:01.478867+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:55:01.478867+0000","last_undegraded":"2024-09-13T14:55:01.478867+0000","last_fullsized":"2024-09-13T14:55:01.478867+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":34,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451942+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:55:00.451942+0000","last_peered":"2024-09-13T14:55:00.451942+0000","last_clean":"2024-09-13T14:55:00.451942+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:55:00.451942+0000","last_undegraded":"2024-09-13T14:55:00.451942+0000","last_fullsized":"2024-09-13T14:55:00.451942+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":431,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.470442+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:55:01.470442+0000","last_peered":"2024-09-13T14:55:01.470442+0000","last_clean":"2024-09-13T14:55:01.470442+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:55:01.470442+0000","last_undegraded":"2024-09-13T14:55:01.470442+0000","last_fullsized":"2024-09-13T14:55:01.470442+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":345,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452416+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:55:00.452416+0000","last_peered":"2024-09-13T14:55:00.452416+0000","last_clean":"2024-09-13T14:55:00.452416+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:55:00.452416+0000","last_undegraded":"2024-09-13T14:55:00.452416+0000","last_fullsized":"2024-09-13T14:55:00.452416+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"291'12603","reported_seq":13380,"reported_epoch":291,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:11.553942+0000","last_change":"2024-09-13T14:54:59.247898+0000","last_active":"2024-09-13T14:55:11.553942+0000","last_peered":"2024-09-13T14:55:11.553942+0000","last_clean":"2024-09-13T14:53:09.541274+0000","last_became_active":"2024-09-13T14:53:11.740035+0000","last_became_peered":"2024-09-13T14:53:11.740035+0000","last_unstale":"2024-09-13T14:55:11.553942+0000","last_undegraded":"2024-09-13T14:53:11.454953+0000","last_fullsized":"2024-09-13T14:53:11.416309+0000","mapping_epoch":271,"log_start":"261'4200","ondisk_log_start":"261'4200","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8403,"log_dups_size":0,"ondisk_log_size":8403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:21.376208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144375808,"num_objects":2203,"num_object_clones":0,"num_object_copies":6609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4362,"num_objects_misplaced":2203,"num_objects_unfound":0,"num_objects_dirty":2203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12603,"num_write_kb":473792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":356,"num_bytes_recovered":23330816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":2203},{"shards":"1,5","objects":-4362}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":22,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.480346+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:55:01.480346+0000","last_peered":"2024-09-13T14:55:01.480346+0000","last_clean":"2024-09-13T14:55:01.480346+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:55:01.480346+0000","last_undegraded":"2024-09-13T14:55:01.480346+0000","last_fullsized":"2024-09-13T14:55:01.480346+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":245,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443640+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:55:00.443640+0000","last_peered":"2024-09-13T14:55:00.443640+0000","last_clean":"2024-09-13T14:55:00.443640+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:55:00.443640+0000","last_undegraded":"2024-09-13T14:55:00.443640+0000","last_fullsized":"2024-09-13T14:55:00.443640+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"291'12866","reported_seq":12965,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:08.253959+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:55:08.253959+0000","last_peered":"2024-09-13T14:55:08.253959+0000","last_clean":"2024-09-13T14:55:08.253959+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:55:08.253959+0000","last_undegraded":"2024-09-13T14:55:08.253959+0000","last_fullsized":"2024-09-13T14:55:08.253959+0000","mapping_epoch":248,"log_start":"265'5868","ondisk_log_start":"265'5868","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:35.987841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12866,"num_write_kb":483200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":80,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.485057+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:55:01.485057+0000","last_peered":"2024-09-13T14:55:01.485057+0000","last_clean":"2024-09-13T14:55:01.485057+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:55:01.485057+0000","last_undegraded":"2024-09-13T14:55:01.485057+0000","last_fullsized":"2024-09-13T14:55:01.485057+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":301,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.461281+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:55:01.461281+0000","last_peered":"2024-09-13T14:55:01.461281+0000","last_clean":"2024-09-13T14:55:01.461281+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:55:01.461281+0000","last_undegraded":"2024-09-13T14:55:01.461281+0000","last_fullsized":"2024-09-13T14:55:01.461281+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"291'12497","reported_seq":12554,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.566388+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:55:11.566388+0000","last_peered":"2024-09-13T14:55:11.566388+0000","last_clean":"2024-09-13T14:55:11.566388+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:55:11.566388+0000","last_undegraded":"2024-09-13T14:55:11.566388+0000","last_fullsized":"2024-09-13T14:55:11.566388+0000","mapping_epoch":244,"log_start":"261'4100","ondisk_log_start":"261'4100","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138739712,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12497,"num_write_kb":467648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":163,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443933+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:55:00.443933+0000","last_peered":"2024-09-13T14:55:00.443933+0000","last_clean":"2024-09-13T14:55:00.443933+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:55:00.443933+0000","last_undegraded":"2024-09-13T14:55:00.443933+0000","last_fullsized":"2024-09-13T14:55:00.443933+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":261,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451278+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:55:00.451278+0000","last_peered":"2024-09-13T14:55:00.451278+0000","last_clean":"2024-09-13T14:55:00.451278+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:55:00.451278+0000","last_undegraded":"2024-09-13T14:55:00.451278+0000","last_fullsized":"2024-09-13T14:55:00.451278+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"291'12708","reported_seq":12765,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.471366+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:55:11.471366+0000","last_peered":"2024-09-13T14:55:11.471366+0000","last_clean":"2024-09-13T14:55:11.471366+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:55:11.471366+0000","last_undegraded":"2024-09-13T14:55:11.471366+0000","last_fullsized":"2024-09-13T14:55:11.471366+0000","mapping_epoch":244,"log_start":"265'6107","ondisk_log_start":"265'6107","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141819904,"num_objects":2164,"num_object_clones":0,"num_object_copies":6492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12708,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":261,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.444104+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:55:00.444104+0000","last_peered":"2024-09-13T14:55:00.444104+0000","last_clean":"2024-09-13T14:55:00.444104+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:55:00.444104+0000","last_undegraded":"2024-09-13T14:55:00.444104+0000","last_fullsized":"2024-09-13T14:55:00.444104+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":104,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.473687+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:55:01.473687+0000","last_peered":"2024-09-13T14:55:01.473687+0000","last_clean":"2024-09-13T14:55:01.473687+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:55:01.473687+0000","last_undegraded":"2024-09-13T14:55:01.473687+0000","last_fullsized":"2024-09-13T14:55:01.473687+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"291'12674","reported_seq":12730,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.530391+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:55:11.530391+0000","last_peered":"2024-09-13T14:55:11.530391+0000","last_clean":"2024-09-13T14:55:11.530391+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:55:11.530391+0000","last_undegraded":"2024-09-13T14:55:11.530391+0000","last_fullsized":"2024-09-13T14:55:11.530391+0000","mapping_epoch":244,"log_start":"265'6145","ondisk_log_start":"265'6145","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139591680,"num_objects":2130,"num_object_clones":0,"num_object_copies":6390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12674,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":755,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.469239+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:55:01.469239+0000","last_peered":"2024-09-13T14:55:01.469239+0000","last_clean":"2024-09-13T14:55:01.469239+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:55:01.469239+0000","last_undegraded":"2024-09-13T14:55:01.469239+0000","last_fullsized":"2024-09-13T14:55:01.469239+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":448,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.467946+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:55:01.467946+0000","last_peered":"2024-09-13T14:55:01.467946+0000","last_clean":"2024-09-13T14:55:01.467946+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:55:01.467946+0000","last_undegraded":"2024-09-13T14:55:01.467946+0000","last_fullsized":"2024-09-13T14:55:01.467946+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":141,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466098+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:55:01.466098+0000","last_peered":"2024-09-13T14:55:01.466098+0000","last_clean":"2024-09-13T14:55:01.466098+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:55:01.466098+0000","last_undegraded":"2024-09-13T14:55:01.466098+0000","last_fullsized":"2024-09-13T14:55:01.466098+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"291'12501","reported_seq":12616,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.753465+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:55:11.753465+0000","last_peered":"2024-09-13T14:55:11.753465+0000","last_clean":"2024-09-13T14:55:11.753465+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:55:11.753465+0000","last_undegraded":"2024-09-13T14:55:11.753465+0000","last_fullsized":"2024-09-13T14:55:11.753465+0000","mapping_epoch":248,"log_start":"261'4100","ondisk_log_start":"261'4100","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8401,"log_dups_size":0,"ondisk_log_size":8401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:06.307356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12501,"num_write_kb":467136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":530,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443523+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:55:00.443523+0000","last_peered":"2024-09-13T14:55:00.443523+0000","last_clean":"2024-09-13T14:55:00.443523+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:55:00.443523+0000","last_undegraded":"2024-09-13T14:55:00.443523+0000","last_fullsized":"2024-09-13T14:55:00.443523+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":300,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451408+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:55:00.451408+0000","last_peered":"2024-09-13T14:55:00.451408+0000","last_clean":"2024-09-13T14:55:00.451408+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:55:00.451408+0000","last_undegraded":"2024-09-13T14:55:00.451408+0000","last_fullsized":"2024-09-13T14:55:00.451408+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":149,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.463089+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:55:01.463089+0000","last_peered":"2024-09-13T14:55:01.463089+0000","last_clean":"2024-09-13T14:55:01.463089+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:55:01.463089+0000","last_undegraded":"2024-09-13T14:55:01.463089+0000","last_fullsized":"2024-09-13T14:55:01.463089+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"291'12507","reported_seq":12573,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.505865+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:55:11.505865+0000","last_peered":"2024-09-13T14:55:11.505865+0000","last_clean":"2024-09-13T14:55:11.505865+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:55:11.505865+0000","last_undegraded":"2024-09-13T14:55:11.505865+0000","last_fullsized":"2024-09-13T14:55:11.505865+0000","mapping_epoch":248,"log_start":"265'5951","ondisk_log_start":"265'5951","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492224,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12507,"num_write_kb":469312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":135,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.461491+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:55:01.461491+0000","last_peered":"2024-09-13T14:55:01.461491+0000","last_clean":"2024-09-13T14:55:01.461491+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:55:01.461491+0000","last_undegraded":"2024-09-13T14:55:01.461491+0000","last_fullsized":"2024-09-13T14:55:01.461491+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452033+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:55:00.452033+0000","last_peered":"2024-09-13T14:55:00.452033+0000","last_clean":"2024-09-13T14:55:00.452033+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:55:00.452033+0000","last_undegraded":"2024-09-13T14:55:00.452033+0000","last_fullsized":"2024-09-13T14:55:00.452033+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"291'12673","reported_seq":12781,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:09.072212+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:55:09.072212+0000","last_peered":"2024-09-13T14:55:09.072212+0000","last_clean":"2024-09-13T14:55:09.072212+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:55:09.072212+0000","last_undegraded":"2024-09-13T14:55:09.072212+0000","last_fullsized":"2024-09-13T14:55:09.072212+0000","mapping_epoch":244,"log_start":"267'6652","ondisk_log_start":"267'6652","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:34.509903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154337280,"num_objects":2355,"num_object_clones":0,"num_object_copies":7065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12673,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":530,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443689+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:55:00.443689+0000","last_peered":"2024-09-13T14:55:00.443689+0000","last_clean":"2024-09-13T14:55:00.443689+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:55:00.443689+0000","last_undegraded":"2024-09-13T14:55:00.443689+0000","last_fullsized":"2024-09-13T14:55:00.443689+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":407,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466316+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:55:01.466316+0000","last_peered":"2024-09-13T14:55:01.466316+0000","last_clean":"2024-09-13T14:55:01.466316+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:55:01.466316+0000","last_undegraded":"2024-09-13T14:55:01.466316+0000","last_fullsized":"2024-09-13T14:55:01.466316+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":413,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.477522+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:55:01.477522+0000","last_peered":"2024-09-13T14:55:01.477522+0000","last_clean":"2024-09-13T14:55:01.477522+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:55:01.477522+0000","last_undegraded":"2024-09-13T14:55:01.477522+0000","last_fullsized":"2024-09-13T14:55:01.477522+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"291'12871","reported_seq":12979,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.271375+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:55:11.271375+0000","last_peered":"2024-09-13T14:55:11.271375+0000","last_clean":"2024-09-13T14:55:11.271375+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:55:11.271375+0000","last_undegraded":"2024-09-13T14:55:11.271375+0000","last_fullsized":"2024-09-13T14:55:11.271375+0000","mapping_epoch":244,"log_start":"267'6546","ondisk_log_start":"267'6546","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6325,"log_dups_size":0,"ondisk_log_size":6325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:34.062885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094400,"num_objects":2275,"num_object_clones":0,"num_object_copies":6825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12871,"num_write_kb":484672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":482,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.480621+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:55:01.480621+0000","last_peered":"2024-09-13T14:55:01.480621+0000","last_clean":"2024-09-13T14:55:01.480621+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:55:01.480621+0000","last_undegraded":"2024-09-13T14:55:01.480621+0000","last_fullsized":"2024-09-13T14:55:01.480621+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":276,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443712+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:55:00.443712+0000","last_peered":"2024-09-13T14:55:00.443712+0000","last_clean":"2024-09-13T14:55:00.443712+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:55:00.443712+0000","last_undegraded":"2024-09-13T14:55:00.443712+0000","last_fullsized":"2024-09-13T14:55:00.443712+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":393,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466169+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:55:01.466169+0000","last_peered":"2024-09-13T14:55:01.466169+0000","last_clean":"2024-09-13T14:55:01.466169+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:55:01.466169+0000","last_undegraded":"2024-09-13T14:55:01.466169+0000","last_fullsized":"2024-09-13T14:55:01.466169+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":256,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452507+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:55:00.452507+0000","last_peered":"2024-09-13T14:55:00.452507+0000","last_clean":"2024-09-13T14:55:00.452507+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:55:00.452507+0000","last_undegraded":"2024-09-13T14:55:00.452507+0000","last_fullsized":"2024-09-13T14:55:00.452507+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"291'12690","reported_seq":15181,"reported_epoch":291,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:11.269491+0000","last_change":"2024-09-13T14:55:00.348097+0000","last_active":"2024-09-13T14:55:11.269491+0000","last_peered":"2024-09-13T14:55:11.269491+0000","last_clean":"2024-09-13T14:54:58.189452+0000","last_became_active":"2024-09-13T14:55:00.316243+0000","last_became_peered":"2024-09-13T14:55:00.316243+0000","last_unstale":"2024-09-13T14:55:11.269491+0000","last_undegraded":"2024-09-13T14:55:11.269491+0000","last_fullsized":"2024-09-13T14:55:11.269491+0000","mapping_epoch":288,"log_start":"266'6351","ondisk_log_start":"266'6351","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6339,"log_dups_size":0,"ondisk_log_size":6339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140771328,"num_objects":2148,"num_object_clones":0,"num_object_copies":6444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2148,"num_objects_unfound":0,"num_objects_dirty":2148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12690,"num_write_kb":474816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":2148}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":676,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478803+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:55:01.478803+0000","last_peered":"2024-09-13T14:55:01.478803+0000","last_clean":"2024-09-13T14:55:01.478803+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:55:01.478803+0000","last_undegraded":"2024-09-13T14:55:01.478803+0000","last_fullsized":"2024-09-13T14:55:01.478803+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":393,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.474773+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:55:01.474773+0000","last_peered":"2024-09-13T14:55:01.474773+0000","last_clean":"2024-09-13T14:55:01.474773+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:55:01.474773+0000","last_undegraded":"2024-09-13T14:55:01.474773+0000","last_fullsized":"2024-09-13T14:55:01.474773+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":104,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.473585+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:55:01.473585+0000","last_peered":"2024-09-13T14:55:01.473585+0000","last_clean":"2024-09-13T14:55:01.473585+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:55:01.473585+0000","last_undegraded":"2024-09-13T14:55:01.473585+0000","last_fullsized":"2024-09-13T14:55:01.473585+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478556+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:55:01.478556+0000","last_peered":"2024-09-13T14:55:01.478556+0000","last_clean":"2024-09-13T14:55:01.478556+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:55:01.478556+0000","last_undegraded":"2024-09-13T14:55:01.478556+0000","last_fullsized":"2024-09-13T14:55:01.478556+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"291'12555","reported_seq":12663,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.217606+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:55:11.217606+0000","last_peered":"2024-09-13T14:55:11.217606+0000","last_clean":"2024-09-13T14:55:11.217606+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:55:11.217606+0000","last_undegraded":"2024-09-13T14:55:11.217606+0000","last_fullsized":"2024-09-13T14:55:11.217606+0000","mapping_epoch":244,"log_start":"266'6232","ondisk_log_start":"266'6232","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6323,"log_dups_size":0,"ondisk_log_size":6323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:23.121414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143327232,"num_objects":2187,"num_object_clones":0,"num_object_copies":6561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12555,"num_write_kb":471744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":594,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.471946+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:55:01.471946+0000","last_peered":"2024-09-13T14:55:01.471946+0000","last_clean":"2024-09-13T14:55:01.471946+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:55:01.471946+0000","last_undegraded":"2024-09-13T14:55:01.471946+0000","last_fullsized":"2024-09-13T14:55:01.471946+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":445,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.461630+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:55:01.461630+0000","last_peered":"2024-09-13T14:55:01.461630+0000","last_clean":"2024-09-13T14:55:01.461630+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:55:01.461630+0000","last_undegraded":"2024-09-13T14:55:01.461630+0000","last_fullsized":"2024-09-13T14:55:01.461630+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":405,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475208+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:55:01.475208+0000","last_peered":"2024-09-13T14:55:01.475208+0000","last_clean":"2024-09-13T14:55:01.475208+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:55:01.475208+0000","last_undegraded":"2024-09-13T14:55:01.475208+0000","last_fullsized":"2024-09-13T14:55:01.475208+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":455,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.477657+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:55:01.477657+0000","last_peered":"2024-09-13T14:55:01.477657+0000","last_clean":"2024-09-13T14:55:01.477657+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:55:01.477657+0000","last_undegraded":"2024-09-13T14:55:01.477657+0000","last_fullsized":"2024-09-13T14:55:01.477657+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"291'12715","reported_seq":12827,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:11.321509+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:55:11.321509+0000","last_peered":"2024-09-13T14:55:11.321509+0000","last_clean":"2024-09-13T14:55:11.321509+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:55:11.321509+0000","last_undegraded":"2024-09-13T14:55:11.321509+0000","last_fullsized":"2024-09-13T14:55:11.321509+0000","mapping_epoch":248,"log_start":"267'6415","ondisk_log_start":"267'6415","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6300,"log_dups_size":0,"ondisk_log_size":6300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:15:37.408790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12715,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":564,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443674+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:55:00.443674+0000","last_peered":"2024-09-13T14:55:00.443674+0000","last_clean":"2024-09-13T14:55:00.443674+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:55:00.443674+0000","last_undegraded":"2024-09-13T14:55:00.443674+0000","last_fullsized":"2024-09-13T14:55:00.443674+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":277,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443850+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:55:00.443850+0000","last_peered":"2024-09-13T14:55:00.443850+0000","last_clean":"2024-09-13T14:55:00.443850+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:55:00.443850+0000","last_undegraded":"2024-09-13T14:55:00.443850+0000","last_fullsized":"2024-09-13T14:55:00.443850+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":301,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.473451+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:55:01.473451+0000","last_peered":"2024-09-13T14:55:01.473451+0000","last_clean":"2024-09-13T14:55:01.473451+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:55:01.473451+0000","last_undegraded":"2024-09-13T14:55:01.473451+0000","last_fullsized":"2024-09-13T14:55:01.473451+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":418,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.477978+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:55:01.477978+0000","last_peered":"2024-09-13T14:55:01.477978+0000","last_clean":"2024-09-13T14:55:01.477978+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:55:01.477978+0000","last_undegraded":"2024-09-13T14:55:01.477978+0000","last_fullsized":"2024-09-13T14:55:01.477978+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"291'12757","reported_seq":16007,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:11.746472+0000","last_change":"2024-09-13T14:54:59.252648+0000","last_active":"2024-09-13T14:55:11.746472+0000","last_peered":"2024-09-13T14:55:11.746472+0000","last_clean":"2024-09-13T14:53:09.553559+0000","last_became_active":"2024-09-13T14:53:11.733490+0000","last_became_peered":"2024-09-13T14:53:11.733490+0000","last_unstale":"2024-09-13T14:55:11.746472+0000","last_undegraded":"2024-09-13T14:53:11.458865+0000","last_fullsized":"2024-09-13T14:53:11.413105+0000","mapping_epoch":271,"log_start":"262'4400","ondisk_log_start":"262'4400","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8357,"log_dups_size":0,"ondisk_log_size":8357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144179224,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12757,"num_write_kb":478593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1569,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":2201},{"shards":"2,6","objects":-3175}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":538,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452212+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:55:00.452212+0000","last_peered":"2024-09-13T14:55:00.452212+0000","last_clean":"2024-09-13T14:55:00.452212+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:55:00.452212+0000","last_undegraded":"2024-09-13T14:55:00.452212+0000","last_fullsized":"2024-09-13T14:55:00.452212+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":277,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.444090+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:55:00.444090+0000","last_peered":"2024-09-13T14:55:00.444090+0000","last_clean":"2024-09-13T14:55:00.444090+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:55:00.444090+0000","last_undegraded":"2024-09-13T14:55:00.444090+0000","last_fullsized":"2024-09-13T14:55:00.444090+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":470,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478519+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:55:01.478519+0000","last_peered":"2024-09-13T14:55:01.478519+0000","last_clean":"2024-09-13T14:55:01.478519+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:55:01.478519+0000","last_undegraded":"2024-09-13T14:55:01.478519+0000","last_fullsized":"2024-09-13T14:55:01.478519+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":333,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.480914+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:55:01.480914+0000","last_peered":"2024-09-13T14:55:01.480914+0000","last_clean":"2024-09-13T14:55:01.480914+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:55:01.480914+0000","last_undegraded":"2024-09-13T14:55:01.480914+0000","last_fullsized":"2024-09-13T14:55:01.480914+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"291'12597","reported_seq":12850,"reported_epoch":291,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:08.951536+0000","last_change":"2024-09-13T14:53:11.783352+0000","last_active":"2024-09-13T14:55:08.951536+0000","last_peered":"2024-09-13T14:55:08.951536+0000","last_clean":"2024-09-13T14:53:09.524137+0000","last_became_active":"2024-09-13T14:53:11.519805+0000","last_became_peered":"2024-09-13T14:53:11.519805+0000","last_unstale":"2024-09-13T14:55:08.951536+0000","last_undegraded":"2024-09-13T14:55:08.951536+0000","last_fullsized":"2024-09-13T14:55:08.951536+0000","mapping_epoch":271,"log_start":"267'6521","ondisk_log_start":"267'6521","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6076,"log_dups_size":0,"ondisk_log_size":6076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2297,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12597,"num_write_kb":477376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4521984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":2321}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":572,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451958+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:55:00.451958+0000","last_peered":"2024-09-13T14:55:00.451958+0000","last_clean":"2024-09-13T14:55:00.451958+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:55:00.451958+0000","last_undegraded":"2024-09-13T14:55:00.451958+0000","last_fullsized":"2024-09-13T14:55:00.451958+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":405,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478383+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:55:01.478383+0000","last_peered":"2024-09-13T14:55:01.478383+0000","last_clean":"2024-09-13T14:55:01.478383+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:55:01.478383+0000","last_undegraded":"2024-09-13T14:55:01.478383+0000","last_fullsized":"2024-09-13T14:55:01.478383+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":104,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.471552+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:55:01.471552+0000","last_peered":"2024-09-13T14:55:01.471552+0000","last_clean":"2024-09-13T14:55:01.471552+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:55:01.471552+0000","last_undegraded":"2024-09-13T14:55:01.471552+0000","last_fullsized":"2024-09-13T14:55:01.471552+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":97,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.473601+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:55:01.473601+0000","last_peered":"2024-09-13T14:55:01.473601+0000","last_clean":"2024-09-13T14:55:01.473601+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:55:01.473601+0000","last_undegraded":"2024-09-13T14:55:01.473601+0000","last_fullsized":"2024-09-13T14:55:01.473601+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"291'12714","reported_seq":12820,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:09.104705+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:55:09.104705+0000","last_peered":"2024-09-13T14:55:09.104705+0000","last_clean":"2024-09-13T14:55:09.104705+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:55:09.104705+0000","last_undegraded":"2024-09-13T14:55:09.104705+0000","last_fullsized":"2024-09-13T14:55:09.104705+0000","mapping_epoch":244,"log_start":"267'6666","ondisk_log_start":"267'6666","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6048,"log_dups_size":0,"ondisk_log_size":6048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:07.929154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147980288,"num_objects":2258,"num_object_clones":0,"num_object_copies":6774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2258,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12714,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":425,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.461106+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:55:01.461106+0000","last_peered":"2024-09-13T14:55:01.461106+0000","last_clean":"2024-09-13T14:55:01.461106+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:55:01.461106+0000","last_undegraded":"2024-09-13T14:55:01.461106+0000","last_fullsized":"2024-09-13T14:55:01.461106+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":415,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.470584+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:55:01.470584+0000","last_peered":"2024-09-13T14:55:01.470584+0000","last_clean":"2024-09-13T14:55:01.470584+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:55:01.470584+0000","last_undegraded":"2024-09-13T14:55:01.470584+0000","last_fullsized":"2024-09-13T14:55:01.470584+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":412,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.485159+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:55:01.485159+0000","last_peered":"2024-09-13T14:55:01.485159+0000","last_clean":"2024-09-13T14:55:01.485159+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:55:01.485159+0000","last_undegraded":"2024-09-13T14:55:01.485159+0000","last_fullsized":"2024-09-13T14:55:01.485159+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":419,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466151+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:55:01.466151+0000","last_peered":"2024-09-13T14:55:01.466151+0000","last_clean":"2024-09-13T14:55:01.466151+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:55:01.466151+0000","last_undegraded":"2024-09-13T14:55:01.466151+0000","last_fullsized":"2024-09-13T14:55:01.466151+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"291'12728","reported_seq":13170,"reported_epoch":291,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:09.083071+0000","last_change":"2024-09-13T14:54:59.249721+0000","last_active":"2024-09-13T14:55:09.083071+0000","last_peered":"2024-09-13T14:55:09.083071+0000","last_clean":"2024-09-13T14:53:09.526490+0000","last_became_active":"2024-09-13T14:53:11.753975+0000","last_became_peered":"2024-09-13T14:53:11.753975+0000","last_unstale":"2024-09-13T14:55:09.083071+0000","last_undegraded":"2024-09-13T14:53:11.475108+0000","last_fullsized":"2024-09-13T14:53:11.413061+0000","mapping_epoch":271,"log_start":"267'6650","ondisk_log_start":"267'6650","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6078,"log_dups_size":0,"ondisk_log_size":6078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147062784,"num_objects":2244,"num_object_clones":0,"num_object_copies":6732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4530,"num_objects_misplaced":2244,"num_objects_unfound":0,"num_objects_dirty":2244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12728,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":11730944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":2244},{"shards":"1,7","objects":-4530}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":665,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466203+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:55:01.466203+0000","last_peered":"2024-09-13T14:55:01.466203+0000","last_clean":"2024-09-13T14:55:01.466203+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:55:01.466203+0000","last_undegraded":"2024-09-13T14:55:01.466203+0000","last_fullsized":"2024-09-13T14:55:01.466203+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":22,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478750+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:55:01.478750+0000","last_peered":"2024-09-13T14:55:01.478750+0000","last_clean":"2024-09-13T14:55:01.478750+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:55:01.478750+0000","last_undegraded":"2024-09-13T14:55:01.478750+0000","last_fullsized":"2024-09-13T14:55:01.478750+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":275,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452337+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:55:00.452337+0000","last_peered":"2024-09-13T14:55:00.452337+0000","last_clean":"2024-09-13T14:55:00.452337+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:55:00.452337+0000","last_undegraded":"2024-09-13T14:55:00.452337+0000","last_fullsized":"2024-09-13T14:55:00.452337+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":34,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443733+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:55:00.443733+0000","last_peered":"2024-09-13T14:55:00.443733+0000","last_clean":"2024-09-13T14:55:00.443733+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:55:00.443733+0000","last_undegraded":"2024-09-13T14:55:00.443733+0000","last_fullsized":"2024-09-13T14:55:00.443733+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":95,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451404+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:55:00.451404+0000","last_peered":"2024-09-13T14:55:00.451404+0000","last_clean":"2024-09-13T14:55:00.451404+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:55:00.451404+0000","last_undegraded":"2024-09-13T14:55:00.451404+0000","last_fullsized":"2024-09-13T14:55:00.451404+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":93,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.451649+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:55:00.451649+0000","last_peered":"2024-09-13T14:55:00.451649+0000","last_clean":"2024-09-13T14:55:00.451649+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:55:00.451649+0000","last_undegraded":"2024-09-13T14:55:00.451649+0000","last_fullsized":"2024-09-13T14:55:00.451649+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":325,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.481118+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:55:01.481118+0000","last_peered":"2024-09-13T14:55:01.481118+0000","last_clean":"2024-09-13T14:55:01.481118+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:55:01.481118+0000","last_undegraded":"2024-09-13T14:55:01.481118+0000","last_fullsized":"2024-09-13T14:55:01.481118+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":364,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.462992+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:55:01.462992+0000","last_peered":"2024-09-13T14:55:01.462992+0000","last_clean":"2024-09-13T14:55:01.462992+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:55:01.462992+0000","last_undegraded":"2024-09-13T14:55:01.462992+0000","last_fullsized":"2024-09-13T14:55:01.462992+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.443969+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:55:00.443969+0000","last_peered":"2024-09-13T14:55:00.443969+0000","last_clean":"2024-09-13T14:55:00.443969+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:55:00.443969+0000","last_undegraded":"2024-09-13T14:55:00.443969+0000","last_fullsized":"2024-09-13T14:55:00.443969+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":392,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.469333+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:55:01.469333+0000","last_peered":"2024-09-13T14:55:01.469333+0000","last_clean":"2024-09-13T14:55:01.469333+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:55:01.469333+0000","last_undegraded":"2024-09-13T14:55:01.469333+0000","last_fullsized":"2024-09-13T14:55:01.469333+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":97,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.493455+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:55:01.493455+0000","last_peered":"2024-09-13T14:55:01.493455+0000","last_clean":"2024-09-13T14:55:01.493455+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:55:01.493455+0000","last_undegraded":"2024-09-13T14:55:01.493455+0000","last_fullsized":"2024-09-13T14:55:01.493455+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":405,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.465764+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:55:01.465764+0000","last_peered":"2024-09-13T14:55:01.465764+0000","last_clean":"2024-09-13T14:55:01.465764+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:55:01.465764+0000","last_undegraded":"2024-09-13T14:55:01.465764+0000","last_fullsized":"2024-09-13T14:55:01.465764+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":262,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452278+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:55:00.452278+0000","last_peered":"2024-09-13T14:55:00.452278+0000","last_clean":"2024-09-13T14:55:00.452278+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:55:00.452278+0000","last_undegraded":"2024-09-13T14:55:00.452278+0000","last_fullsized":"2024-09-13T14:55:00.452278+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":449,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466579+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:55:01.466579+0000","last_peered":"2024-09-13T14:55:01.466579+0000","last_clean":"2024-09-13T14:55:01.466579+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:55:01.466579+0000","last_undegraded":"2024-09-13T14:55:01.466579+0000","last_fullsized":"2024-09-13T14:55:01.466579+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":97,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.479007+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:55:01.479007+0000","last_peered":"2024-09-13T14:55:01.479007+0000","last_clean":"2024-09-13T14:55:01.479007+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:55:01.479007+0000","last_undegraded":"2024-09-13T14:55:01.479007+0000","last_fullsized":"2024-09-13T14:55:01.479007+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":410,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.481492+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:55:01.481492+0000","last_peered":"2024-09-13T14:55:01.481492+0000","last_clean":"2024-09-13T14:55:01.481492+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:55:01.481492+0000","last_undegraded":"2024-09-13T14:55:01.481492+0000","last_fullsized":"2024-09-13T14:55:01.481492+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":411,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.465982+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:55:01.465982+0000","last_peered":"2024-09-13T14:55:01.465982+0000","last_clean":"2024-09-13T14:55:01.465982+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:55:01.465982+0000","last_undegraded":"2024-09-13T14:55:01.465982+0000","last_fullsized":"2024-09-13T14:55:01.465982+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":69,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.481268+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:55:01.481268+0000","last_peered":"2024-09-13T14:55:01.481268+0000","last_clean":"2024-09-13T14:55:01.481268+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:55:01.481268+0000","last_undegraded":"2024-09-13T14:55:01.481268+0000","last_fullsized":"2024-09-13T14:55:01.481268+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":431,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.464746+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:55:01.464746+0000","last_peered":"2024-09-13T14:55:01.464746+0000","last_clean":"2024-09-13T14:55:01.464746+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:55:01.464746+0000","last_undegraded":"2024-09-13T14:55:01.464746+0000","last_fullsized":"2024-09-13T14:55:01.464746+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":18,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.473394+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:55:01.473394+0000","last_peered":"2024-09-13T14:55:01.473394+0000","last_clean":"2024-09-13T14:55:01.473394+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:55:01.473394+0000","last_undegraded":"2024-09-13T14:55:01.473394+0000","last_fullsized":"2024-09-13T14:55:01.473394+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":155,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.467865+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:55:01.467865+0000","last_peered":"2024-09-13T14:55:01.467865+0000","last_clean":"2024-09-13T14:55:01.467865+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:55:01.467865+0000","last_undegraded":"2024-09-13T14:55:01.467865+0000","last_fullsized":"2024-09-13T14:55:01.467865+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":274,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.444033+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:55:00.444033+0000","last_peered":"2024-09-13T14:55:00.444033+0000","last_clean":"2024-09-13T14:55:00.444033+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:55:00.444033+0000","last_undegraded":"2024-09-13T14:55:00.444033+0000","last_fullsized":"2024-09-13T14:55:00.444033+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466276+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:55:01.466276+0000","last_peered":"2024-09-13T14:55:01.466276+0000","last_clean":"2024-09-13T14:55:01.466276+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:55:01.466276+0000","last_undegraded":"2024-09-13T14:55:01.466276+0000","last_fullsized":"2024-09-13T14:55:01.466276+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":259,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.444025+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:55:00.444025+0000","last_peered":"2024-09-13T14:55:00.444025+0000","last_clean":"2024-09-13T14:55:00.444025+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:55:00.444025+0000","last_undegraded":"2024-09-13T14:55:00.444025+0000","last_fullsized":"2024-09-13T14:55:00.444025+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":451,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.469892+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:55:01.469892+0000","last_peered":"2024-09-13T14:55:01.469892+0000","last_clean":"2024-09-13T14:55:01.469892+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:55:01.469892+0000","last_undegraded":"2024-09-13T14:55:01.469892+0000","last_fullsized":"2024-09-13T14:55:01.469892+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":427,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.466475+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:55:01.466475+0000","last_peered":"2024-09-13T14:55:01.466475+0000","last_clean":"2024-09-13T14:55:01.466475+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:55:01.466475+0000","last_undegraded":"2024-09-13T14:55:01.466475+0000","last_fullsized":"2024-09-13T14:55:01.466475+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":62,"reported_epoch":290,"state":"active+clean","last_fresh":"2024-09-13T14:55:00.452261+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:55:00.452261+0000","last_peered":"2024-09-13T14:55:00.452261+0000","last_clean":"2024-09-13T14:55:00.452261+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:55:00.452261+0000","last_undegraded":"2024-09-13T14:55:00.452261+0000","last_fullsized":"2024-09-13T14:55:00.452261+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":24,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.480190+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:55:01.480190+0000","last_peered":"2024-09-13T14:55:01.480190+0000","last_clean":"2024-09-13T14:55:01.480190+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:55:01.480190+0000","last_undegraded":"2024-09-13T14:55:01.480190+0000","last_fullsized":"2024-09-13T14:55:01.480190+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2316763160,"num_objects":35352,"num_object_clones":0,"num_object_copies":106056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14802,"num_objects_misplaced":8892,"num_objects_unfound":0,"num_objects_dirty":35352,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202656,"num_write_kb":7616193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3355,"num_bytes_recovered":219676672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7870164992,"data_stored":7870152776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109752,"ondisk_log_size":109752,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234421,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1817720,"kb_used_data":843952,"kb_used_omap":1,"kb_used_meta":973694,"kb_avail":91927432,"statfs":{"total":95995035648,"available":94133690368,"internally_reserved":0,"allocated":864206848,"data_stored":862617097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":997063115},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496059,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022908,"kb_used_data":1125492,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":91722244,"statfs":{"total":95995035648,"available":93923577856,"internally_reserved":0,"allocated":1152503808,"data_stored":1150909982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":918944199},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757697,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1636976,"kb_used_data":568936,"kb_used_omap":1,"kb_used_meta":1068030,"kb_avail":92108176,"statfs":{"total":95995035648,"available":94318772224,"internally_reserved":0,"allocated":582590464,"data_stored":581004929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1093663174},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019333,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650545,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551208,"kb_used_data":992160,"kb_used_omap":4,"kb_used_meta":558971,"kb_avail":92193944,"statfs":{"total":95995035648,"available":94406598656,"internally_reserved":0,"allocated":1015971840,"data_stored":1014394237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":572386863},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879480,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1281216,"kb_used_data":840824,"kb_used_omap":6,"kb_used_meta":440377,"kb_avail":92463936,"statfs":{"total":95995035648,"available":94683070464,"internally_reserved":0,"allocated":861003776,"data_stored":859406993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":450946617},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435496,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2098656,"kb_used_data":1326552,"kb_used_omap":5,"kb_used_meta":772090,"kb_avail":91646496,"statfs":{"total":95995035648,"available":93846011904,"internally_reserved":0,"allocated":1358389248,"data_stored":1356818698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":790620513},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,2,3,3,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926071,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":987308,"kb_used_data":594788,"kb_used_omap":5,"kb_used_meta":392506,"kb_avail":92757844,"statfs":{"total":95995035648,"available":94984032256,"internally_reserved":0,"allocated":609062912,"data_stored":607481273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":401926314},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,6,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":599654400,"data_stored":599654400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1340669952,"data_stored":1340669952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":847384576,"data_stored":847380504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1010302976,"data_stored":1010302976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1509625856,"data_stored":1509621784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":574357504,"data_stored":574357504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1139019776,"data_stored":1139015704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":849149952,"data_stored":849149952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:15.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:15.962 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:15.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:15.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:16.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:16.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:16.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:16.221 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:16.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.400 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:16.425 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:16.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:16.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[113926]: pgmap v1017: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 14915/103293 objects degraded (14.440%); 8512/103293 objects misplaced (8.241%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14915/103293 objects degraded (14.440%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[108570]: pgmap v1017: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 14915/103293 objects degraded (14.440%); 8512/103293 objects misplaced (8.241%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14915/103293 objects degraded (14.440%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:16.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:16 smithi159 ceph-mon[102519]: pgmap v1017: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 14915/103293 objects degraded (14.440%); 8512/103293 objects misplaced (8.241%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14915/103293 objects degraded (14.440%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:16.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:16.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:16.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:16.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:16.958 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:16.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:17.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:17.083 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:17.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:17.186 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.223 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:17.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:17.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.352 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:17.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:17.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.483 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:17.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.585 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:17.586 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:17 smithi159 ceph-mon[102519]: pgmap v1018: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14915/100917 objects degraded (14.779%); 8331/100917 objects misplaced (8.255%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:17.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:17 smithi042 ceph-mon[113926]: pgmap v1018: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14915/100917 objects degraded (14.779%); 8331/100917 objects misplaced (8.255%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:17.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:17 smithi042 ceph-mon[108570]: pgmap v1018: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 14915/100917 objects degraded (14.779%); 8331/100917 objects misplaced (8.255%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:17.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:17.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:17.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:17.756 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:17.858 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:17.885 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:17.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:17.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:55:17.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.011 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:18.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:18.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:18.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:18.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:18.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:18.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:18.554 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:18.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:18.699 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:18.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:18.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:55:18.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:18.849 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:18.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:18.902 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:18.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:18.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:18.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:19.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.128 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:19.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:19.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:19.359 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:19.406 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:19.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:19.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:19.531 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:19.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:19.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:19.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.670 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:19.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:19.771 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:19.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:19.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:19 smithi042 ceph-mon[113926]: pgmap v1019: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14992/96078 objects degraded (15.604%); 8133/96078 objects misplaced (8.465%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:19 smithi042 ceph-mon[108570]: pgmap v1019: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14992/96078 objects degraded (15.604%); 8133/96078 objects misplaced (8.465%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:19.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:19] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.51.0" 2024-09-13T14:55:19.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:19.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:19.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:19.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:19 smithi159 ceph-mon[102519]: pgmap v1019: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 14992/96078 objects degraded (15.604%); 8133/96078 objects misplaced (8.465%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:20.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:20.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:20.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:20.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:20.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:20.449 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:20.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:20.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:20.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:20.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:20.768 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:20.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14992/96078 objects degraded (15.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:20.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14992/96078 objects degraded (15.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:20.870 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:20.912 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:20.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14992/96078 objects degraded (15.604%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:21.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:21.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:21.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:21.166 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:21.191 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:21.211 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:21.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:21.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:21.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:21.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:21.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:21.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:21.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:21.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:21.710 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:21.812 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:21 smithi042 ceph-mon[113926]: pgmap v1020: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 15108/93654 objects degraded (16.132%); 7733/93654 objects misplaced (8.257%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:21.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:21 smithi042 ceph-mon[108570]: pgmap v1020: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 15108/93654 objects degraded (16.132%); 7733/93654 objects misplaced (8.257%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:21.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:21.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:21.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:21.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:22.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:21 smithi159 ceph-mon[102519]: pgmap v1020: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.9 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 15108/93654 objects degraded (16.132%); 7733/93654 objects misplaced (8.257%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:22.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:22.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:22.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:22.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:22.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:22.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:22.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:22.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:22.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:22.537 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:22.539 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:22.556 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:55:22.556 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:22.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:22.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:22.587 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:55:22.588 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:22.588 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2668165114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:22.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2668165114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:22.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:22.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:22.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:22.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:22.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:22.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:22.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:22.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2668165114' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:23.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:23.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:23.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:23.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:23.350 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:23.452 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:23.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:23.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:23.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:23.764 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:23.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:23 smithi042 ceph-mon[108570]: pgmap v1021: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 15173/87048 objects degraded (17.431%); 7331/87048 objects misplaced (8.422%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:23.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:23 smithi042 ceph-mon[113926]: pgmap v1021: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 15173/87048 objects degraded (17.431%); 7331/87048 objects misplaced (8.422%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:23.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:23.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:23.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:23.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:23 smithi159 ceph-mon[102519]: pgmap v1021: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.8 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 15173/87048 objects degraded (17.431%); 7331/87048 objects misplaced (8.422%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:24.011 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:24.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:24.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:24.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:24.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.309 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:24.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:24.410 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:24.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:24.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:24.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:24.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:24.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:24.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:24.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:24.880 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:24.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:24.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:24.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.042 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:25.167 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:25.269 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:25.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:25.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:25.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:25.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.584 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:25.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:25.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:25 smithi042 ceph-mon[108570]: pgmap v1022: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15330/84051 objects degraded (18.239%); 6904/84051 objects misplaced (8.214%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15330/84051 objects degraded (18.239%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:25 smithi042 ceph-mon[113926]: pgmap v1022: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15330/84051 objects degraded (18.239%); 6904/84051 objects misplaced (8.214%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15330/84051 objects degraded (18.239%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:25.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:25.994 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:25.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:25.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:25 smithi159 ceph-mon[102519]: pgmap v1022: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15330/84051 objects degraded (18.239%); 6904/84051 objects misplaced (8.214%); 2.4 MiB/s, 38 objects/s recovering 2024-09-13T14:55:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15330/84051 objects degraded (18.239%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:26.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:26.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:26.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:26.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:26.352 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.391 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:26.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:26.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:26.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:26.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:26.644 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:26.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:26.746 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:26.769 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:26.871 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:26.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:26.915 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:26.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:26.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.059 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:27.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:27.078 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:27.079 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:27.161 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:27.187 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:27.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:27.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:27.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:27.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:27.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:27.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.618 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:27.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:27.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:27 smithi159 ceph-mon[102519]: pgmap v1023: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15330/81666 objects degraded (18.772%); 6716/81666 objects misplaced (8.224%); 2.1 MiB/s, 32 objects/s recovering 2024-09-13T14:55:27.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:27.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:27.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:27 smithi042 ceph-mon[108570]: pgmap v1023: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15330/81666 objects degraded (18.772%); 6716/81666 objects misplaced (8.224%); 2.1 MiB/s, 32 objects/s recovering 2024-09-13T14:55:27.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:27 smithi042 ceph-mon[113926]: pgmap v1023: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 600 op/s; 15330/81666 objects degraded (18.772%); 6716/81666 objects misplaced (8.224%); 2.1 MiB/s, 32 objects/s recovering 2024-09-13T14:55:27.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:27.884 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:27.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:27.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:28.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:28.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:28.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:28.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:28.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:28.432 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:28.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:28.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:28.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:28.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:28.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:28.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:28.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:28.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:28.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:55:28.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:28.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:28.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:28.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:29.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:29.140 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:29.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.249 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:29.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:29.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:29.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:29.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.672 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:29.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:29 smithi042 ceph-mon[113926]: pgmap v1024: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 15421/76737 objects degraded (20.096%); 6500/76737 objects misplaced (8.470%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:29.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:29 smithi042 ceph-mon[108570]: pgmap v1024: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 15421/76737 objects degraded (20.096%); 6500/76737 objects misplaced (8.470%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:29.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:29.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:29.905 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:29 smithi159 ceph-mon[102519]: pgmap v1024: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 15421/76737 objects degraded (20.096%); 6500/76737 objects misplaced (8.470%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:29.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:29.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:29.984 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:29.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:30.086 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:29] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.51.0" 2024-09-13T14:55:30.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:30.194 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:55:30.196 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:55:30.231 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:30.270 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:30.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:30.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:30.412 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:30.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:30.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:30.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:30.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:30.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:30.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:30.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:30 smithi159 ceph-mon[102519]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:30.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15421/76737 objects degraded (20.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:30.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:30.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:30.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:30 smithi042 ceph-mon[113926]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:30.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15421/76737 objects degraded (20.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:30.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:30 smithi042 ceph-mon[108570]: from='client.55126 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:30.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15421/76737 objects degraded (20.096%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:30.910 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:30.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:30.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:30.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:31.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:31.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:31.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.423 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:31.423 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.425 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:31.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:31.565 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:31.667 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:31.715 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:31 smithi159 ceph-mon[102519]: pgmap v1025: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 15535/74181 objects degraded (20.942%); 6090/74181 objects misplaced (8.210%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:31.809 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:31.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:31.840 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:31.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:31.855 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:31 smithi042 ceph-mon[108570]: pgmap v1025: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 15535/74181 objects degraded (20.942%); 6090/74181 objects misplaced (8.210%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:31.855 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:31 smithi042 ceph-mon[113926]: pgmap v1025: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 15535/74181 objects degraded (20.942%); 6090/74181 objects misplaced (8.210%); 2.5 MiB/s, 39 objects/s recovering 2024-09-13T14:55:31.856 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1024,"stamp":"2024-09-13T14:55:28.468503+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1675171837,"num_objects":25579,"num_object_clones":0,"num_object_copies":76737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15421,"num_objects_misplaced":6500,"num_objects_unfound":0,"num_objects_dirty":25579,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212519,"num_write_kb":7616226,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3995,"num_bytes_recovered":258545160,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111449,"ondisk_log_size":111449,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9637000,"kb_used_data":4544720,"kb_used_omap":27,"kb_used_meta":5092068,"kb_avail":646579064,"statfs":{"total":671965249536,"available":662096961536,"internally_reserved":0,"allocated":4653793280,"data_stored":4642642202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5214278603},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-528220160,"num_objects":-8060,"num_object_clones":0,"num_object_copies":-24180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":506,"num_objects_misplaced":-1831,"num_objects_unfound":0,"num_objects_dirty":-8060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8060,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":519,"num_bytes_recovered":34013184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005186"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":407,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299633+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:55:14.299633+0000","last_peered":"2024-09-13T14:55:14.299633+0000","last_clean":"2024-09-13T14:55:14.299633+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:55:14.299633+0000","last_undegraded":"2024-09-13T14:55:14.299633+0000","last_fullsized":"2024-09-13T14:55:14.299633+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":73,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.286226+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:55:14.286226+0000","last_peered":"2024-09-13T14:55:14.286226+0000","last_clean":"2024-09-13T14:55:14.286226+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:55:14.286226+0000","last_undegraded":"2024-09-13T14:55:14.286226+0000","last_fullsized":"2024-09-13T14:55:14.286226+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":260,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478724+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:55:01.478724+0000","last_peered":"2024-09-13T14:55:01.478724+0000","last_clean":"2024-09-13T14:55:01.478724+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:55:01.478724+0000","last_undegraded":"2024-09-13T14:55:01.478724+0000","last_fullsized":"2024-09-13T14:55:01.478724+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":99,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.303766+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:55:14.303766+0000","last_peered":"2024-09-13T14:55:14.303766+0000","last_clean":"2024-09-13T14:55:14.303766+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:55:14.303766+0000","last_undegraded":"2024-09-13T14:55:14.303766+0000","last_fullsized":"2024-09-13T14:55:14.303766+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.317681+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:55:14.317681+0000","last_peered":"2024-09-13T14:55:14.317681+0000","last_clean":"2024-09-13T14:55:14.317681+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:55:14.317681+0000","last_undegraded":"2024-09-13T14:55:14.317681+0000","last_fullsized":"2024-09-13T14:55:14.317681+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":35,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475468+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:55:01.475468+0000","last_peered":"2024-09-13T14:55:01.475468+0000","last_clean":"2024-09-13T14:55:01.475468+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:55:01.475468+0000","last_undegraded":"2024-09-13T14:55:01.475468+0000","last_fullsized":"2024-09-13T14:55:01.475468+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":433,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.300655+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:55:14.300655+0000","last_peered":"2024-09-13T14:55:14.300655+0000","last_clean":"2024-09-13T14:55:14.300655+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:55:14.300655+0000","last_undegraded":"2024-09-13T14:55:14.300655+0000","last_fullsized":"2024-09-13T14:55:14.300655+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":346,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475127+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:55:01.475127+0000","last_peered":"2024-09-13T14:55:01.475127+0000","last_clean":"2024-09-13T14:55:01.475127+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:55:01.475127+0000","last_undegraded":"2024-09-13T14:55:01.475127+0000","last_fullsized":"2024-09-13T14:55:01.475127+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"292'13208","reported_seq":14735,"reported_epoch":292,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:26.593157+0000","last_change":"2024-09-13T14:54:59.247898+0000","last_active":"2024-09-13T14:55:26.593157+0000","last_peered":"2024-09-13T14:55:26.593157+0000","last_clean":"2024-09-13T14:53:09.541274+0000","last_became_active":"2024-09-13T14:53:11.740035+0000","last_became_peered":"2024-09-13T14:53:11.740035+0000","last_unstale":"2024-09-13T14:55:26.593157+0000","last_undegraded":"2024-09-13T14:53:11.454953+0000","last_fullsized":"2024-09-13T14:53:11.416309+0000","mapping_epoch":271,"log_start":"262'4800","ondisk_log_start":"262'4800","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8408,"log_dups_size":0,"ondisk_log_size":8408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:21.376208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104726528,"num_objects":1598,"num_object_clones":0,"num_object_copies":4794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4595,"num_objects_misplaced":1596,"num_objects_unfound":0,"num_objects_dirty":1598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13208,"num_write_kb":473792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":730,"num_bytes_recovered":47841280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1596},{"shards":"1,5","objects":-4593}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":24,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.315822+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:55:14.315822+0000","last_peered":"2024-09-13T14:55:14.315822+0000","last_clean":"2024-09-13T14:55:14.315822+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:55:14.315822+0000","last_undegraded":"2024-09-13T14:55:14.315822+0000","last_fullsized":"2024-09-13T14:55:14.315822+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":247,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.300645+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:55:14.300645+0000","last_peered":"2024-09-13T14:55:14.300645+0000","last_clean":"2024-09-13T14:55:14.300645+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:55:14.300645+0000","last_undegraded":"2024-09-13T14:55:14.300645+0000","last_fullsized":"2024-09-13T14:55:14.300645+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"292'13645","reported_seq":13747,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:28.326317+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:55:28.326317+0000","last_peered":"2024-09-13T14:55:28.326317+0000","last_clean":"2024-09-13T14:55:28.326317+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:55:28.326317+0000","last_undegraded":"2024-09-13T14:55:28.326317+0000","last_fullsized":"2024-09-13T14:55:28.326317+0000","mapping_epoch":248,"log_start":"267'6668","ondisk_log_start":"267'6668","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:35.987841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95354880,"num_objects":1455,"num_object_clones":0,"num_object_copies":4365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13645,"num_write_kb":483200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":82,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.292728+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:55:14.292728+0000","last_peered":"2024-09-13T14:55:14.292728+0000","last_clean":"2024-09-13T14:55:14.292728+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:55:14.292728+0000","last_undegraded":"2024-09-13T14:55:14.292728+0000","last_fullsized":"2024-09-13T14:55:14.292728+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.285547+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:55:14.285547+0000","last_peered":"2024-09-13T14:55:14.285547+0000","last_clean":"2024-09-13T14:55:14.285547+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:55:14.285547+0000","last_undegraded":"2024-09-13T14:55:14.285547+0000","last_fullsized":"2024-09-13T14:55:14.285547+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"292'13079","reported_seq":13138,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.606530+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:55:26.606530+0000","last_peered":"2024-09-13T14:55:26.606530+0000","last_clean":"2024-09-13T14:55:26.606530+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:55:26.606530+0000","last_undegraded":"2024-09-13T14:55:26.606530+0000","last_fullsized":"2024-09-13T14:55:26.606530+0000","mapping_epoch":244,"log_start":"262'4700","ondisk_log_start":"262'4700","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8379,"log_dups_size":0,"ondisk_log_size":8379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100597760,"num_objects":1535,"num_object_clones":0,"num_object_copies":4605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13079,"num_write_kb":467648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":164,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478645+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:55:01.478645+0000","last_peered":"2024-09-13T14:55:01.478645+0000","last_clean":"2024-09-13T14:55:01.478645+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:55:01.478645+0000","last_undegraded":"2024-09-13T14:55:01.478645+0000","last_fullsized":"2024-09-13T14:55:01.478645+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":262,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475573+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:55:01.475573+0000","last_peered":"2024-09-13T14:55:01.475573+0000","last_clean":"2024-09-13T14:55:01.475573+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:55:01.475573+0000","last_undegraded":"2024-09-13T14:55:01.475573+0000","last_fullsized":"2024-09-13T14:55:01.475573+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"292'13339","reported_seq":13398,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.556695+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:55:26.556695+0000","last_peered":"2024-09-13T14:55:26.556695+0000","last_clean":"2024-09-13T14:55:26.556695+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:55:26.556695+0000","last_undegraded":"2024-09-13T14:55:26.556695+0000","last_fullsized":"2024-09-13T14:55:26.556695+0000","mapping_epoch":244,"log_start":"267'6807","ondisk_log_start":"267'6807","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100466688,"num_objects":1533,"num_object_clones":0,"num_object_copies":4599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13339,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":262,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478601+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:55:01.478601+0000","last_peered":"2024-09-13T14:55:01.478601+0000","last_clean":"2024-09-13T14:55:01.478601+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:55:01.478601+0000","last_undegraded":"2024-09-13T14:55:01.478601+0000","last_fullsized":"2024-09-13T14:55:01.478601+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.302822+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:55:14.302822+0000","last_peered":"2024-09-13T14:55:14.302822+0000","last_clean":"2024-09-13T14:55:14.302822+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:55:14.302822+0000","last_undegraded":"2024-09-13T14:55:14.302822+0000","last_fullsized":"2024-09-13T14:55:14.302822+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"292'13264","reported_seq":13322,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.543498+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:55:26.543498+0000","last_peered":"2024-09-13T14:55:26.543498+0000","last_clean":"2024-09-13T14:55:26.543498+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:55:26.543498+0000","last_undegraded":"2024-09-13T14:55:26.543498+0000","last_fullsized":"2024-09-13T14:55:26.543498+0000","mapping_epoch":244,"log_start":"267'6645","ondisk_log_start":"267'6645","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100925440,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13264,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":757,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.305305+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:55:14.305305+0000","last_peered":"2024-09-13T14:55:14.305305+0000","last_clean":"2024-09-13T14:55:14.305305+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:55:14.305305+0000","last_undegraded":"2024-09-13T14:55:14.305305+0000","last_fullsized":"2024-09-13T14:55:14.305305+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":450,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.306713+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:55:14.306713+0000","last_peered":"2024-09-13T14:55:14.306713+0000","last_clean":"2024-09-13T14:55:14.306713+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:55:14.306713+0000","last_undegraded":"2024-09-13T14:55:14.306713+0000","last_fullsized":"2024-09-13T14:55:14.306713+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":144,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299884+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:55:14.299884+0000","last_peered":"2024-09-13T14:55:14.299884+0000","last_clean":"2024-09-13T14:55:14.299884+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:55:14.299884+0000","last_undegraded":"2024-09-13T14:55:14.299884+0000","last_fullsized":"2024-09-13T14:55:14.299884+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"292'13097","reported_seq":13215,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.808272+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:55:26.808272+0000","last_peered":"2024-09-13T14:55:26.808272+0000","last_clean":"2024-09-13T14:55:26.808272+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:55:26.808272+0000","last_undegraded":"2024-09-13T14:55:26.808272+0000","last_fullsized":"2024-09-13T14:55:26.808272+0000","mapping_epoch":248,"log_start":"262'4700","ondisk_log_start":"262'4700","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8397,"log_dups_size":0,"ondisk_log_size":8397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:06.307356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13097,"num_write_kb":467136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":531,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478126+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:55:01.478126+0000","last_peered":"2024-09-13T14:55:01.478126+0000","last_clean":"2024-09-13T14:55:01.478126+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:55:01.478126+0000","last_undegraded":"2024-09-13T14:55:01.478126+0000","last_fullsized":"2024-09-13T14:55:01.478126+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":301,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475647+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:55:01.475647+0000","last_peered":"2024-09-13T14:55:01.475647+0000","last_clean":"2024-09-13T14:55:01.475647+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:55:01.475647+0000","last_undegraded":"2024-09-13T14:55:01.475647+0000","last_fullsized":"2024-09-13T14:55:01.475647+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":152,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.288728+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:55:14.288728+0000","last_peered":"2024-09-13T14:55:14.288728+0000","last_clean":"2024-09-13T14:55:14.288728+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:55:14.288728+0000","last_undegraded":"2024-09-13T14:55:14.288728+0000","last_fullsized":"2024-09-13T14:55:14.288728+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"292'13134","reported_seq":13201,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.547503+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:55:26.547503+0000","last_peered":"2024-09-13T14:55:26.547503+0000","last_clean":"2024-09-13T14:55:26.547503+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:55:26.547503+0000","last_undegraded":"2024-09-13T14:55:26.547503+0000","last_fullsized":"2024-09-13T14:55:26.547503+0000","mapping_epoch":248,"log_start":"267'6551","ondisk_log_start":"267'6551","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13134,"num_write_kb":469312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":138,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.285727+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:55:14.285727+0000","last_peered":"2024-09-13T14:55:14.285727+0000","last_clean":"2024-09-13T14:55:14.285727+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:55:14.285727+0000","last_undegraded":"2024-09-13T14:55:14.285727+0000","last_fullsized":"2024-09-13T14:55:14.285727+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.279341+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:55:14.279341+0000","last_peered":"2024-09-13T14:55:14.279341+0000","last_clean":"2024-09-13T14:55:14.279341+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:55:14.279341+0000","last_undegraded":"2024-09-13T14:55:14.279341+0000","last_fullsized":"2024-09-13T14:55:14.279341+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"292'13254","reported_seq":13365,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:24.141777+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:55:24.141777+0000","last_peered":"2024-09-13T14:55:24.141777+0000","last_clean":"2024-09-13T14:55:24.141777+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:55:24.141777+0000","last_undegraded":"2024-09-13T14:55:24.141777+0000","last_fullsized":"2024-09-13T14:55:24.141777+0000","mapping_epoch":244,"log_start":"268'7252","ondisk_log_start":"268'7252","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6002,"log_dups_size":0,"ondisk_log_size":6002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:34.509903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116260864,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13254,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":531,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478207+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:55:01.478207+0000","last_peered":"2024-09-13T14:55:01.478207+0000","last_clean":"2024-09-13T14:55:01.478207+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:55:01.478207+0000","last_undegraded":"2024-09-13T14:55:01.478207+0000","last_fullsized":"2024-09-13T14:55:01.478207+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":409,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.306226+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:55:14.306226+0000","last_peered":"2024-09-13T14:55:14.306226+0000","last_clean":"2024-09-13T14:55:14.306226+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:55:14.306226+0000","last_undegraded":"2024-09-13T14:55:14.306226+0000","last_fullsized":"2024-09-13T14:55:14.306226+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":415,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.305826+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:55:14.305826+0000","last_peered":"2024-09-13T14:55:14.305826+0000","last_clean":"2024-09-13T14:55:14.305826+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:55:14.305826+0000","last_undegraded":"2024-09-13T14:55:14.305826+0000","last_fullsized":"2024-09-13T14:55:14.305826+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"292'13471","reported_seq":13581,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.306921+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:55:26.306921+0000","last_peered":"2024-09-13T14:55:26.306921+0000","last_clean":"2024-09-13T14:55:26.306921+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:55:26.306921+0000","last_undegraded":"2024-09-13T14:55:26.306921+0000","last_fullsized":"2024-09-13T14:55:26.306921+0000","mapping_epoch":244,"log_start":"268'7146","ondisk_log_start":"268'7146","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6325,"log_dups_size":0,"ondisk_log_size":6325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:34.062885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109772800,"num_objects":1675,"num_object_clones":0,"num_object_copies":5025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13471,"num_write_kb":484672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":484,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.297221+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:55:14.297221+0000","last_peered":"2024-09-13T14:55:14.297221+0000","last_clean":"2024-09-13T14:55:14.297221+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:55:14.297221+0000","last_undegraded":"2024-09-13T14:55:14.297221+0000","last_fullsized":"2024-09-13T14:55:14.297221+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":277,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478590+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:55:01.478590+0000","last_peered":"2024-09-13T14:55:01.478590+0000","last_clean":"2024-09-13T14:55:01.478590+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:55:01.478590+0000","last_undegraded":"2024-09-13T14:55:01.478590+0000","last_fullsized":"2024-09-13T14:55:01.478590+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":395,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.306986+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:55:14.306986+0000","last_peered":"2024-09-13T14:55:14.306986+0000","last_clean":"2024-09-13T14:55:14.306986+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:55:14.306986+0000","last_undegraded":"2024-09-13T14:55:14.306986+0000","last_fullsized":"2024-09-13T14:55:14.306986+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":257,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475734+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:55:01.475734+0000","last_peered":"2024-09-13T14:55:01.475734+0000","last_clean":"2024-09-13T14:55:01.475734+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:55:01.475734+0000","last_undegraded":"2024-09-13T14:55:01.475734+0000","last_fullsized":"2024-09-13T14:55:01.475734+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"292'13268","reported_seq":15761,"reported_epoch":292,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:26.372831+0000","last_change":"2024-09-13T14:55:00.348097+0000","last_active":"2024-09-13T14:55:26.372831+0000","last_peered":"2024-09-13T14:55:26.372831+0000","last_clean":"2024-09-13T14:54:58.189452+0000","last_became_active":"2024-09-13T14:55:00.316243+0000","last_became_peered":"2024-09-13T14:55:00.316243+0000","last_unstale":"2024-09-13T14:55:26.372831+0000","last_undegraded":"2024-09-13T14:55:26.372831+0000","last_fullsized":"2024-09-13T14:55:26.372831+0000","mapping_epoch":288,"log_start":"268'6951","ondisk_log_start":"268'6951","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6317,"log_dups_size":0,"ondisk_log_size":6317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102891520,"num_objects":1570,"num_object_clones":0,"num_object_copies":4710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1570,"num_objects_unfound":0,"num_objects_dirty":1570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13268,"num_write_kb":474816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":1570}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":678,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290518+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:55:14.290518+0000","last_peered":"2024-09-13T14:55:14.290518+0000","last_clean":"2024-09-13T14:55:14.290518+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:55:14.290518+0000","last_undegraded":"2024-09-13T14:55:14.290518+0000","last_fullsized":"2024-09-13T14:55:14.290518+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":395,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.289793+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:55:14.289793+0000","last_peered":"2024-09-13T14:55:14.289793+0000","last_clean":"2024-09-13T14:55:14.289793+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:55:14.289793+0000","last_undegraded":"2024-09-13T14:55:14.289793+0000","last_fullsized":"2024-09-13T14:55:14.289793+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":106,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.308373+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:55:14.308373+0000","last_peered":"2024-09-13T14:55:14.308373+0000","last_clean":"2024-09-13T14:55:14.308373+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:55:14.308373+0000","last_undegraded":"2024-09-13T14:55:14.308373+0000","last_fullsized":"2024-09-13T14:55:14.308373+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":20,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.304094+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:55:14.304094+0000","last_peered":"2024-09-13T14:55:14.304094+0000","last_clean":"2024-09-13T14:55:14.304094+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:55:14.304094+0000","last_undegraded":"2024-09-13T14:55:14.304094+0000","last_fullsized":"2024-09-13T14:55:14.304094+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"292'13176","reported_seq":13286,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.289970+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:55:26.289970+0000","last_peered":"2024-09-13T14:55:26.289970+0000","last_clean":"2024-09-13T14:55:26.289970+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:55:26.289970+0000","last_undegraded":"2024-09-13T14:55:26.289970+0000","last_fullsized":"2024-09-13T14:55:26.289970+0000","mapping_epoch":244,"log_start":"268'6832","ondisk_log_start":"268'6832","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:23.121414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13176,"num_write_kb":471744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":596,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.304206+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:55:14.304206+0000","last_peered":"2024-09-13T14:55:14.304206+0000","last_clean":"2024-09-13T14:55:14.304206+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:55:14.304206+0000","last_undegraded":"2024-09-13T14:55:14.304206+0000","last_fullsized":"2024-09-13T14:55:14.304206+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":447,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.285806+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:55:14.285806+0000","last_peered":"2024-09-13T14:55:14.285806+0000","last_clean":"2024-09-13T14:55:14.285806+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:55:14.285806+0000","last_undegraded":"2024-09-13T14:55:14.285806+0000","last_fullsized":"2024-09-13T14:55:14.285806+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":407,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290136+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:55:14.290136+0000","last_peered":"2024-09-13T14:55:14.290136+0000","last_clean":"2024-09-13T14:55:14.290136+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:55:14.290136+0000","last_undegraded":"2024-09-13T14:55:14.290136+0000","last_fullsized":"2024-09-13T14:55:14.290136+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":457,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.305703+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:55:14.305703+0000","last_peered":"2024-09-13T14:55:14.305703+0000","last_clean":"2024-09-13T14:55:14.305703+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:55:14.305703+0000","last_undegraded":"2024-09-13T14:55:14.305703+0000","last_fullsized":"2024-09-13T14:55:14.305703+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"292'13341","reported_seq":13458,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:26.355842+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:55:26.355842+0000","last_peered":"2024-09-13T14:55:26.355842+0000","last_clean":"2024-09-13T14:55:26.355842+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:55:26.355842+0000","last_undegraded":"2024-09-13T14:55:26.355842+0000","last_fullsized":"2024-09-13T14:55:26.355842+0000","mapping_epoch":248,"log_start":"268'7015","ondisk_log_start":"268'7015","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6326,"log_dups_size":0,"ondisk_log_size":6326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:15:37.408790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107020288,"num_objects":1633,"num_object_clones":0,"num_object_copies":4899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13341,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":566,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299597+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:55:14.299597+0000","last_peered":"2024-09-13T14:55:14.299597+0000","last_clean":"2024-09-13T14:55:14.299597+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:55:14.299597+0000","last_undegraded":"2024-09-13T14:55:14.299597+0000","last_fullsized":"2024-09-13T14:55:14.299597+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":278,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478465+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:55:01.478465+0000","last_peered":"2024-09-13T14:55:01.478465+0000","last_clean":"2024-09-13T14:55:01.478465+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:55:01.478465+0000","last_undegraded":"2024-09-13T14:55:01.478465+0000","last_fullsized":"2024-09-13T14:55:01.478465+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.302597+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:55:14.302597+0000","last_peered":"2024-09-13T14:55:14.302597+0000","last_clean":"2024-09-13T14:55:14.302597+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:55:14.302597+0000","last_undegraded":"2024-09-13T14:55:14.302597+0000","last_fullsized":"2024-09-13T14:55:14.302597+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":420,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290811+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:55:14.290811+0000","last_peered":"2024-09-13T14:55:14.290811+0000","last_clean":"2024-09-13T14:55:14.290811+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:55:14.290811+0000","last_undegraded":"2024-09-13T14:55:14.290811+0000","last_fullsized":"2024-09-13T14:55:14.290811+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"292'13348","reported_seq":16602,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:26.786101+0000","last_change":"2024-09-13T14:54:59.252648+0000","last_active":"2024-09-13T14:55:26.786101+0000","last_peered":"2024-09-13T14:55:26.786101+0000","last_clean":"2024-09-13T14:53:09.553559+0000","last_became_active":"2024-09-13T14:53:11.733490+0000","last_became_peered":"2024-09-13T14:53:11.733490+0000","last_unstale":"2024-09-13T14:55:26.786101+0000","last_undegraded":"2024-09-13T14:53:11.458865+0000","last_fullsized":"2024-09-13T14:53:11.413105+0000","mapping_epoch":271,"log_start":"262'5000","ondisk_log_start":"262'5000","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8348,"log_dups_size":0,"ondisk_log_size":8348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447448,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13348,"num_write_kb":478593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1569,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":1610},{"shards":"2,6","objects":-3766}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":539,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475181+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:55:01.475181+0000","last_peered":"2024-09-13T14:55:01.475181+0000","last_clean":"2024-09-13T14:55:01.475181+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:55:01.475181+0000","last_undegraded":"2024-09-13T14:55:01.475181+0000","last_fullsized":"2024-09-13T14:55:01.475181+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":278,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478772+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:55:01.478772+0000","last_peered":"2024-09-13T14:55:01.478772+0000","last_clean":"2024-09-13T14:55:01.478772+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:55:01.478772+0000","last_undegraded":"2024-09-13T14:55:01.478772+0000","last_fullsized":"2024-09-13T14:55:01.478772+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":472,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.291610+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:55:14.291610+0000","last_peered":"2024-09-13T14:55:14.291610+0000","last_clean":"2024-09-13T14:55:14.291610+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:55:14.291610+0000","last_undegraded":"2024-09-13T14:55:14.291610+0000","last_fullsized":"2024-09-13T14:55:14.291610+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":335,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290193+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:55:14.290193+0000","last_peered":"2024-09-13T14:55:14.290193+0000","last_clean":"2024-09-13T14:55:14.290193+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:55:14.290193+0000","last_undegraded":"2024-09-13T14:55:14.290193+0000","last_fullsized":"2024-09-13T14:55:14.290193+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"292'13204","reported_seq":13459,"reported_epoch":292,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:24.087204+0000","last_change":"2024-09-13T14:53:11.783352+0000","last_active":"2024-09-13T14:55:24.087204+0000","last_peered":"2024-09-13T14:55:24.087204+0000","last_clean":"2024-09-13T14:53:09.524137+0000","last_became_active":"2024-09-13T14:53:11.519805+0000","last_became_peered":"2024-09-13T14:53:11.519805+0000","last_unstale":"2024-09-13T14:55:24.087204+0000","last_undegraded":"2024-09-13T14:55:24.087204+0000","last_fullsized":"2024-09-13T14:55:24.087204+0000","mapping_epoch":271,"log_start":"268'7121","ondisk_log_start":"268'7121","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112328704,"num_objects":1714,"num_object_clones":0,"num_object_copies":5142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1695,"num_objects_unfound":0,"num_objects_dirty":1714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13204,"num_write_kb":477376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4521984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":1714}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":573,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475181+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:55:01.475181+0000","last_peered":"2024-09-13T14:55:01.475181+0000","last_clean":"2024-09-13T14:55:01.475181+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:55:01.475181+0000","last_undegraded":"2024-09-13T14:55:01.475181+0000","last_fullsized":"2024-09-13T14:55:01.475181+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":407,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.291500+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:55:14.291500+0000","last_peered":"2024-09-13T14:55:14.291500+0000","last_clean":"2024-09-13T14:55:14.291500+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:55:14.291500+0000","last_undegraded":"2024-09-13T14:55:14.291500+0000","last_fullsized":"2024-09-13T14:55:14.291500+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":106,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.303969+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:55:14.303969+0000","last_peered":"2024-09-13T14:55:14.303969+0000","last_clean":"2024-09-13T14:55:14.303969+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:55:14.303969+0000","last_undegraded":"2024-09-13T14:55:14.303969+0000","last_fullsized":"2024-09-13T14:55:14.303969+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":99,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.302729+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:55:14.302729+0000","last_peered":"2024-09-13T14:55:14.302729+0000","last_clean":"2024-09-13T14:55:14.302729+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:55:14.302729+0000","last_undegraded":"2024-09-13T14:55:14.302729+0000","last_fullsized":"2024-09-13T14:55:14.302729+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"292'13295","reported_seq":13405,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:24.203551+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:55:24.203551+0000","last_peered":"2024-09-13T14:55:24.203551+0000","last_clean":"2024-09-13T14:55:24.203551+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:55:24.203551+0000","last_undegraded":"2024-09-13T14:55:24.203551+0000","last_fullsized":"2024-09-13T14:55:24.203551+0000","mapping_epoch":244,"log_start":"268'7266","ondisk_log_start":"268'7266","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6029,"log_dups_size":0,"ondisk_log_size":6029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:07.929154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13295,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":428,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.288430+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:55:14.288430+0000","last_peered":"2024-09-13T14:55:14.288430+0000","last_clean":"2024-09-13T14:55:14.288430+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:55:14.288430+0000","last_undegraded":"2024-09-13T14:55:14.288430+0000","last_fullsized":"2024-09-13T14:55:14.288430+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":417,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.300867+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:55:14.300867+0000","last_peered":"2024-09-13T14:55:14.300867+0000","last_clean":"2024-09-13T14:55:14.300867+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:55:14.300867+0000","last_undegraded":"2024-09-13T14:55:14.300867+0000","last_fullsized":"2024-09-13T14:55:14.300867+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":414,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.293027+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:55:14.293027+0000","last_peered":"2024-09-13T14:55:14.293027+0000","last_clean":"2024-09-13T14:55:14.293027+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:55:14.293027+0000","last_undegraded":"2024-09-13T14:55:14.293027+0000","last_fullsized":"2024-09-13T14:55:14.293027+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":421,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.303705+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:55:14.303705+0000","last_peered":"2024-09-13T14:55:14.303705+0000","last_clean":"2024-09-13T14:55:14.303705+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:55:14.303705+0000","last_undegraded":"2024-09-13T14:55:14.303705+0000","last_fullsized":"2024-09-13T14:55:14.303705+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"292'13332","reported_seq":14215,"reported_epoch":292,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:24.167630+0000","last_change":"2024-09-13T14:54:59.249721+0000","last_active":"2024-09-13T14:55:24.167630+0000","last_peered":"2024-09-13T14:55:24.167630+0000","last_clean":"2024-09-13T14:53:09.526490+0000","last_became_active":"2024-09-13T14:53:11.753975+0000","last_became_peered":"2024-09-13T14:53:11.753975+0000","last_unstale":"2024-09-13T14:55:24.167630+0000","last_undegraded":"2024-09-13T14:53:11.475108+0000","last_fullsized":"2024-09-13T14:53:11.413061+0000","mapping_epoch":271,"log_start":"268'7250","ondisk_log_start":"268'7250","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6082,"log_dups_size":0,"ondisk_log_size":6082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107479040,"num_objects":1640,"num_object_clones":0,"num_object_copies":4920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4916,"num_objects_misplaced":1639,"num_objects_unfound":0,"num_objects_dirty":1640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13332,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":400,"num_bytes_recovered":26083328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1639},{"shards":"1,7","objects":-4915}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":667,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299466+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:55:14.299466+0000","last_peered":"2024-09-13T14:55:14.299466+0000","last_clean":"2024-09-13T14:55:14.299466+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:55:14.299466+0000","last_undegraded":"2024-09-13T14:55:14.299466+0000","last_fullsized":"2024-09-13T14:55:14.299466+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":24,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.304346+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:55:14.304346+0000","last_peered":"2024-09-13T14:55:14.304346+0000","last_clean":"2024-09-13T14:55:14.304346+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:55:14.304346+0000","last_undegraded":"2024-09-13T14:55:14.304346+0000","last_fullsized":"2024-09-13T14:55:14.304346+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":276,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475321+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:55:01.475321+0000","last_peered":"2024-09-13T14:55:01.475321+0000","last_clean":"2024-09-13T14:55:01.475321+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:55:01.475321+0000","last_undegraded":"2024-09-13T14:55:01.475321+0000","last_fullsized":"2024-09-13T14:55:01.475321+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":36,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299700+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:55:14.299700+0000","last_peered":"2024-09-13T14:55:14.299700+0000","last_clean":"2024-09-13T14:55:14.299700+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:55:14.299700+0000","last_undegraded":"2024-09-13T14:55:14.299700+0000","last_fullsized":"2024-09-13T14:55:14.299700+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":96,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475933+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:55:01.475933+0000","last_peered":"2024-09-13T14:55:01.475933+0000","last_clean":"2024-09-13T14:55:01.475933+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:55:01.475933+0000","last_undegraded":"2024-09-13T14:55:01.475933+0000","last_fullsized":"2024-09-13T14:55:01.475933+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":94,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475760+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:55:01.475760+0000","last_peered":"2024-09-13T14:55:01.475760+0000","last_clean":"2024-09-13T14:55:01.475760+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:55:01.475760+0000","last_undegraded":"2024-09-13T14:55:01.475760+0000","last_fullsized":"2024-09-13T14:55:01.475760+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":328,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290456+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:55:14.290456+0000","last_peered":"2024-09-13T14:55:14.290456+0000","last_clean":"2024-09-13T14:55:14.290456+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:55:14.290456+0000","last_undegraded":"2024-09-13T14:55:14.290456+0000","last_fullsized":"2024-09-13T14:55:14.290456+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":366,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.288605+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:55:14.288605+0000","last_peered":"2024-09-13T14:55:14.288605+0000","last_clean":"2024-09-13T14:55:14.288605+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:55:14.288605+0000","last_undegraded":"2024-09-13T14:55:14.288605+0000","last_fullsized":"2024-09-13T14:55:14.288605+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":261,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478569+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:55:01.478569+0000","last_peered":"2024-09-13T14:55:01.478569+0000","last_clean":"2024-09-13T14:55:01.478569+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:55:01.478569+0000","last_undegraded":"2024-09-13T14:55:01.478569+0000","last_fullsized":"2024-09-13T14:55:01.478569+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":394,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.305408+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:55:14.305408+0000","last_peered":"2024-09-13T14:55:14.305408+0000","last_clean":"2024-09-13T14:55:14.305408+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:55:14.305408+0000","last_undegraded":"2024-09-13T14:55:14.305408+0000","last_fullsized":"2024-09-13T14:55:14.305408+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":99,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.316245+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:55:14.316245+0000","last_peered":"2024-09-13T14:55:14.316245+0000","last_clean":"2024-09-13T14:55:14.316245+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:55:14.316245+0000","last_undegraded":"2024-09-13T14:55:14.316245+0000","last_fullsized":"2024-09-13T14:55:14.316245+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":407,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.303207+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:55:14.303207+0000","last_peered":"2024-09-13T14:55:14.303207+0000","last_clean":"2024-09-13T14:55:14.303207+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:55:14.303207+0000","last_undegraded":"2024-09-13T14:55:14.303207+0000","last_fullsized":"2024-09-13T14:55:14.303207+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":263,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.475253+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:55:01.475253+0000","last_peered":"2024-09-13T14:55:01.475253+0000","last_clean":"2024-09-13T14:55:01.475253+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:55:01.475253+0000","last_undegraded":"2024-09-13T14:55:01.475253+0000","last_fullsized":"2024-09-13T14:55:01.475253+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":451,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.304393+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:55:14.304393+0000","last_peered":"2024-09-13T14:55:14.304393+0000","last_clean":"2024-09-13T14:55:14.304393+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:55:14.304393+0000","last_undegraded":"2024-09-13T14:55:14.304393+0000","last_fullsized":"2024-09-13T14:55:14.304393+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":100,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.318117+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:55:14.318117+0000","last_peered":"2024-09-13T14:55:14.318117+0000","last_clean":"2024-09-13T14:55:14.318117+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:55:14.318117+0000","last_undegraded":"2024-09-13T14:55:14.318117+0000","last_fullsized":"2024-09-13T14:55:14.318117+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":413,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290852+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:55:14.290852+0000","last_peered":"2024-09-13T14:55:14.290852+0000","last_clean":"2024-09-13T14:55:14.290852+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:55:14.290852+0000","last_undegraded":"2024-09-13T14:55:14.290852+0000","last_fullsized":"2024-09-13T14:55:14.290852+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":413,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.299952+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:55:14.299952+0000","last_peered":"2024-09-13T14:55:14.299952+0000","last_clean":"2024-09-13T14:55:14.299952+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:55:14.299952+0000","last_undegraded":"2024-09-13T14:55:14.299952+0000","last_fullsized":"2024-09-13T14:55:14.299952+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":71,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.290551+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:55:14.290551+0000","last_peered":"2024-09-13T14:55:14.290551+0000","last_clean":"2024-09-13T14:55:14.290551+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:55:14.290551+0000","last_undegraded":"2024-09-13T14:55:14.290551+0000","last_fullsized":"2024-09-13T14:55:14.290551+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":433,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.287102+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:55:14.287102+0000","last_peered":"2024-09-13T14:55:14.287102+0000","last_clean":"2024-09-13T14:55:14.287102+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:55:14.287102+0000","last_undegraded":"2024-09-13T14:55:14.287102+0000","last_fullsized":"2024-09-13T14:55:14.287102+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":20,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.302421+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:55:14.302421+0000","last_peered":"2024-09-13T14:55:14.302421+0000","last_clean":"2024-09-13T14:55:14.302421+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:55:14.302421+0000","last_undegraded":"2024-09-13T14:55:14.302421+0000","last_fullsized":"2024-09-13T14:55:14.302421+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.306619+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:55:14.306619+0000","last_peered":"2024-09-13T14:55:14.306619+0000","last_clean":"2024-09-13T14:55:14.306619+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:55:14.306619+0000","last_undegraded":"2024-09-13T14:55:14.306619+0000","last_fullsized":"2024-09-13T14:55:14.306619+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":275,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478646+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:55:01.478646+0000","last_peered":"2024-09-13T14:55:01.478646+0000","last_clean":"2024-09-13T14:55:01.478646+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:55:01.478646+0000","last_undegraded":"2024-09-13T14:55:01.478646+0000","last_fullsized":"2024-09-13T14:55:01.478646+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":295,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.307166+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:55:14.307166+0000","last_peered":"2024-09-13T14:55:14.307166+0000","last_clean":"2024-09-13T14:55:14.307166+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:55:14.307166+0000","last_undegraded":"2024-09-13T14:55:14.307166+0000","last_fullsized":"2024-09-13T14:55:14.307166+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":260,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.478756+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:55:01.478756+0000","last_peered":"2024-09-13T14:55:01.478756+0000","last_clean":"2024-09-13T14:55:01.478756+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:55:01.478756+0000","last_undegraded":"2024-09-13T14:55:01.478756+0000","last_fullsized":"2024-09-13T14:55:01.478756+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":453,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.306111+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:55:14.306111+0000","last_peered":"2024-09-13T14:55:14.306111+0000","last_clean":"2024-09-13T14:55:14.306111+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:55:14.306111+0000","last_undegraded":"2024-09-13T14:55:14.306111+0000","last_fullsized":"2024-09-13T14:55:14.306111+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":429,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.300481+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:55:14.300481+0000","last_peered":"2024-09-13T14:55:14.300481+0000","last_clean":"2024-09-13T14:55:14.300481+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:55:14.300481+0000","last_undegraded":"2024-09-13T14:55:14.300481+0000","last_fullsized":"2024-09-13T14:55:14.300481+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":63,"reported_epoch":291,"state":"active+clean","last_fresh":"2024-09-13T14:55:01.474797+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:55:01.474797+0000","last_peered":"2024-09-13T14:55:01.474797+0000","last_clean":"2024-09-13T14:55:01.474797+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:55:01.474797+0000","last_undegraded":"2024-09-13T14:55:01.474797+0000","last_fullsized":"2024-09-13T14:55:01.474797+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":26,"reported_epoch":292,"state":"active+clean","last_fresh":"2024-09-13T14:55:14.315679+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:55:14.315679+0000","last_peered":"2024-09-13T14:55:14.315679+0000","last_clean":"2024-09-13T14:55:14.315679+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:55:14.315679+0000","last_undegraded":"2024-09-13T14:55:14.315679+0000","last_fullsized":"2024-09-13T14:55:14.315679+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1674575896,"num_objects":25553,"num_object_clones":0,"num_object_copies":76659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15421,"num_objects_misplaced":6500,"num_objects_unfound":0,"num_objects_dirty":25553,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212455,"num_write_kb":7616193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3948,"num_bytes_recovered":258539520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6042300416,"data_stored":6042288200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109751,"ondisk_log_size":109751,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234424,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1585224,"kb_used_data":611520,"kb_used_omap":1,"kb_used_meta":973694,"kb_avail":92159928,"statfs":{"total":95995035648,"available":94371766272,"internally_reserved":0,"allocated":626196480,"data_stored":624598816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":997063115},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496062,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1716492,"kb_used_data":814980,"kb_used_omap":1,"kb_used_meta":901502,"kb_avail":92028660,"statfs":{"total":95995035648,"available":94237347840,"internally_reserved":0,"allocated":834539520,"data_stored":832937781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":923138503},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757700,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1474496,"kb_used_data":414648,"kb_used_omap":1,"kb_used_meta":1059838,"kb_avail":92270656,"statfs":{"total":95995035648,"available":94485151744,"internally_reserved":0,"allocated":424599552,"data_stored":423006104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1085274566},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019336,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650548,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1281336,"kb_used_data":722288,"kb_used_omap":4,"kb_used_meta":558971,"kb_avail":92463816,"statfs":{"total":95995035648,"available":94682947584,"internally_reserved":0,"allocated":739622912,"data_stored":738037396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":572386863},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879483,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1041680,"kb_used_data":605448,"kb_used_omap":6,"kb_used_meta":436217,"kb_avail":92703472,"statfs":{"total":95995035648,"available":94928355328,"internally_reserved":0,"allocated":619978752,"data_stored":618374056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":446686777},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435499,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1762992,"kb_used_data":985256,"kb_used_omap":5,"kb_used_meta":777658,"kb_avail":91982160,"statfs":{"total":95995035648,"available":94189731840,"internally_reserved":0,"allocated":1008902144,"data_stored":1007323681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":796322145},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926075,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":774780,"kb_used_data":390580,"kb_used_omap":5,"kb_used_meta":384186,"kb_avail":92970372,"statfs":{"total":95995035648,"available":95201660928,"internally_reserved":0,"allocated":399953920,"data_stored":398364368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":393406634},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":388431872,"data_stored":388431872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":989986816,"data_stored":989986816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":609947648,"data_stored":609943576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":733216768,"data_stored":733216768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1468731392,"data_stored":1468727320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":416546816,"data_stored":416546816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":822874112,"data_stored":822870040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":612564992,"data_stored":612564992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:31.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:55:31.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:31.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:31.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:32.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:32.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:32.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:32.148 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:55:32.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:32.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:32.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:32.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:32.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:32.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:32.437 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:32.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:32.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:32.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:32.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:32.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:32.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:32.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:32.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:32.909 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:32.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:32.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:32.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:33.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:33.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:33.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:33.350 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:33.452 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:33.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:33.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:33.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:33.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:33.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:33.716 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:33.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.757 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:33.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:33.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[108570]: pgmap v1026: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 15609/67353 objects degraded (23.175%); 5668/67353 objects misplaced (8.415%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:33.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/965469010' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:33.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:33.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[113926]: pgmap v1026: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 15609/67353 objects degraded (23.175%); 5668/67353 objects misplaced (8.415%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:33.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/965469010' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:33.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:33.858 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:33.893 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:33.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:33.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:33.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:33 smithi159 ceph-mon[102519]: pgmap v1026: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.4 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 15609/67353 objects degraded (23.175%); 5668/67353 objects misplaced (8.415%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:55:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/965469010' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:34.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:34.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:34.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:34.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:34.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:34.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:34.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:34.403 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:55:34.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:34.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:34.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:34.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:34.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:34.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:34.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:34 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:34 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:34.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:34 smithi159 ceph-mon[102519]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:55:34.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:34.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:34.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:34.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:34.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[108570]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:55:34.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:55:34.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:55:34.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:34 smithi042 ceph-mon[113926]: osdmap e293: 8 total, 8 up, 7 in 2024-09-13T14:55:34.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:34.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:34.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:34.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:35.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:35.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:35.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:35.129 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:35.129 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:35.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:35.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:35.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:35.362 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:35.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:35.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:35.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:35.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:35.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:35.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:35.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:35.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:35.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:35.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:35.836 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:35 smithi159 ceph-mon[102519]: pgmap v1028: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15728/64374 objects degraded (24.432%); 5292/64374 objects misplaced (8.221%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:35.836 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15728/64374 objects degraded (24.432%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:35.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:35.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:55:35.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:35.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:35.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:36.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:36.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:35 smithi042 ceph-mon[113926]: pgmap v1028: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15728/64374 objects degraded (24.432%); 5292/64374 objects misplaced (8.221%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15728/64374 objects degraded (24.432%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:35 smithi042 ceph-mon[108570]: pgmap v1028: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15728/64374 objects degraded (24.432%); 5292/64374 objects misplaced (8.221%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15728/64374 objects degraded (24.432%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:36.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:36.123 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:36.145 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:36.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:36.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:36.271 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:36.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:36.414 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:36.516 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:36.542 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:36.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:36.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:36.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:36.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:36.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:36.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:36.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:37.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:37.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:37.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:37.303 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:37.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:37.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:37.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:37.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:37.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:37.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:37.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:37.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:37.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:37.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:37.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:37.879 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:37.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:37.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:37.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:38.008 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:38.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:38.027 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:38.028 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:38.110 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.182 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:38.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:38.283 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:38.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:38.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:38 smithi159 ceph-mon[102519]: pgmap v1029: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 15728/61884 objects degraded (25.415%); 5090/61884 objects misplaced (8.225%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:38.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:38 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.384 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:38 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:38.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:38.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:38.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:38.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[108570]: pgmap v1029: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 15728/61884 objects degraded (25.415%); 5090/61884 objects misplaced (8.225%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:38.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.559 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[113926]: pgmap v1029: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 659 op/s; 15728/61884 objects degraded (25.415%); 5090/61884 objects misplaced (8.225%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:55:38.559 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.559 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:38 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:38.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:38.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.589 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:38.600 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:38.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:38.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:38.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:38.763 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:55:38.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:38.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:38.888 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:38.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:38.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.011 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:39.049 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:39.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:39.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:39.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:39.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:39.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:39.553 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:39.554 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:39.554 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:39.554 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:55:39.554 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:55:39.554 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:39.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:39.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:39.654 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:55:39.654 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:39.699 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:55:39.699 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:39.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:39.735 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:55:39.736 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:39.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:55:39.772 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:39.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:39.807 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:39] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-13T14:55:39.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:55:39.808 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:39.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:55:39.852 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:55:39.852 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:39.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:39.899 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:55:39.899 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:39.936 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:55:39.936 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:39.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:39.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:40.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:40.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:40.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:40.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:40.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:40.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.497 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:40 smithi159 ceph-mon[102519]: pgmap v1030: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15814/57087 objects degraded (27.702%); 4875/57087 objects misplaced (8.540%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1513904125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:40 smithi042 ceph-mon[113926]: pgmap v1030: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15814/57087 objects degraded (27.702%); 4875/57087 objects misplaced (8.540%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1513904125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:40 smithi042 ceph-mon[108570]: pgmap v1030: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.2 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 15814/57087 objects degraded (27.702%); 4875/57087 objects misplaced (8.540%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1513904125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:40.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:40.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:40.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:40.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:40.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:40.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:40.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:41.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:41.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:41.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:41.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.227 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:41.337 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15814/57087 objects degraded (27.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:41.338 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15814/57087 objects degraded (27.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:41.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:41.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:41.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:41.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:41.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15814/57087 objects degraded (27.702%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:41.583 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:41.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:41.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:41.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:41.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:41.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:41.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:41.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:41.997 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:42.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:42.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:42.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:42.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:42.287 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:42.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:42.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:42.414 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:42.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:42 smithi159 ceph-mon[102519]: pgmap v1031: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15933/54510 objects degraded (29.229%); 4474/54510 objects misplaced (8.208%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:42.501 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:42 smithi042 ceph-mon[113926]: pgmap v1031: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15933/54510 objects degraded (29.229%); 4474/54510 objects misplaced (8.208%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:42.502 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:42 smithi042 ceph-mon[108570]: pgmap v1031: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 1.1 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 15933/54510 objects degraded (29.229%); 4474/54510 objects misplaced (8.208%); 2.6 MiB/s, 40 objects/s recovering 2024-09-13T14:55:42.516 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:42.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:42.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:42.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.665 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:42.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:42.805 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:42.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:42.890 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:55:42.890 INFO:teuthology.orchestra.run.smithi042.stdout:{"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15933/54510 objects degraded (29.229%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1507,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":293,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238436,"num_in_osds":7,"osd_in_since":1726239189,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":94},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":18170,"data_bytes":1189615613,"bytes_used":8673751040,"bytes_avail":663291498496,"bytes_total":671965249536,"degraded_objects":15933,"degraded_total":54510,"degraded_ratio":0.29229499174463403,"misplaced_objects":4474,"misplaced_total":54510,"misplaced_ratio":0.082076683177398649,"recovering_objects_per_sec":40,"recovering_bytes_per_sec":2685870,"recovering_keys_per_sec":0,"num_objects_recovered":410,"num_bytes_recovered":26869760,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":655},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi042.pkkbnw","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-13T14:50:44.357827+0000","services":{}},"progress_events":{}} 2024-09-13T14:55:42.907 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:42.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:42.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:42.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:43.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:43.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:43.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:43.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:55:43.471 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:43.493 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:43.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:43.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2931837674' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:55:43.524 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2931837674' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:55:43.524 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2931837674' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:55:43.596 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:43.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:43.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:43.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:43.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:43.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:43.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:43.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:43.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:43.916 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:55:43.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:44.016 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:44.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:44.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:44.050 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:44.065 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:44.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:44.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:44.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:44.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.371 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:44.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:44.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:44 smithi159 ceph-mon[102519]: pgmap v1032: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 998 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15994/47943 objects degraded (33.360%); 4082/47943 objects misplaced (8.514%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:55:44.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:44.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:44 smithi042 ceph-mon[108570]: pgmap v1032: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 998 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15994/47943 objects degraded (33.360%); 4082/47943 objects misplaced (8.514%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:55:44.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:44 smithi042 ceph-mon[113926]: pgmap v1032: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 998 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 15994/47943 objects degraded (33.360%); 4082/47943 objects misplaced (8.514%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:55:44.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:44.623 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:44.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:44.748 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:44.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:44.902 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:44.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:44.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:45.004 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.026 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.127 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:45.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.253 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:45.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:45.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:45.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:45.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:45.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:45.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:45.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:45.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:45.776 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:45.820 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:45.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:45.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:45.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:45.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:46.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.089 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:46.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:46.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:46.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:46.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:46.483 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:46 smithi159 ceph-mon[102519]: pgmap v1033: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 936 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 16074/44991 objects degraded (35.727%); 3693/44991 objects misplaced (8.208%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:55:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:46 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16074/44991 objects degraded (35.727%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:46.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:46 smithi042 ceph-mon[108570]: pgmap v1033: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 936 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 16074/44991 objects degraded (35.727%); 3693/44991 objects misplaced (8.208%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:55:46.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:46 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16074/44991 objects degraded (35.727%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:46.550 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:46 smithi042 ceph-mon[113926]: pgmap v1033: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 936 MiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 16074/44991 objects degraded (35.727%); 3693/44991 objects misplaced (8.208%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:55:46.550 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:46 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16074/44991 objects degraded (35.727%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:46.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:46.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.754 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:46.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:46.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:46.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:46.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:46.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:47.039 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:47.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:47.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:47.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:47.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:47.364 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:47.466 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:47.497 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:47.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:47.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:47.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:47.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:47.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:47.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:47.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:47.921 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:47.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:47.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:48.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:48.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:48.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:48.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:48.255 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:55:48.296 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:48.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:48.543 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:48.565 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[113926]: pgmap v1034: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 16074/42636 objects degraded (37.701%); 3500/42636 objects misplaced (8.209%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:55:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/13714674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[108570]: pgmap v1034: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 16074/42636 objects degraded (37.701%); 3500/42636 objects misplaced (8.209%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:55:48.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/13714674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:48.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:48.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:48.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:48 smithi159 ceph-mon[102519]: pgmap v1034: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 887 MiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 603 op/s; 16074/42636 objects degraded (37.701%); 3500/42636 objects misplaced (8.209%); 2.2 MiB/s, 35 objects/s recovering 2024-09-13T14:55:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/13714674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:55:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:55:48.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:55:48.771 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:55:48.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:55:48.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:55:48.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:55:48.772 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:55:48.772 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:55:48.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:48.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:48.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:48.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:48.946 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:48.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:49.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:49.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:49.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:49.211 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:49.211 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:49.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:49.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:49.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:49.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:49.491 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:49 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:49 smithi159 ceph-mon[102519]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:55:49.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:49.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[113926]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:55:49.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:55:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2941856157' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:55:49.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-mon[108570]: osdmap e294: 8 total, 8 up, 7 in 2024-09-13T14:55:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.639 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:49.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:49.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:49.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:49.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:49.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:49.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:50.007 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:50.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:55:50.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:49] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-13T14:55:50.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:50.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:50.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:50.356 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:50.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.357 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:50.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:50.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:50.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:50.527 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:50 smithi159 ceph-mon[102519]: pgmap v1036: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 805 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16115/38709 objects degraded (41.631%); 3312/38709 objects misplaced (8.556%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:55:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:50 smithi042 ceph-mon[113926]: pgmap v1036: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 805 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16115/38709 objects degraded (41.631%); 3312/38709 objects misplaced (8.556%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:55:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:50 smithi042 ceph-mon[108570]: pgmap v1036: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 805 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16115/38709 objects degraded (41.631%); 3312/38709 objects misplaced (8.556%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:55:50.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:50.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:50.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:50.789 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:50.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:50.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:50.913 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:50.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:50.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:51.016 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:51.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:51.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:51.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:51.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:51.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:51.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:51.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:51 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16115/38709 objects degraded (41.631%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.583 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16115/38709 objects degraded (41.631%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16115/38709 objects degraded (41.631%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:55:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:55:51.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:51.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.747 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:51.849 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:51.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:51.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:51.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.002 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:52.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:52.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:52.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:52.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:52.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:52.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:52.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:52.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:52.435 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:52 smithi159 ceph-mon[102519]: pgmap v1037: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 751 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16201/36093 objects degraded (44.887%); 2942/36093 objects misplaced (8.151%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:55:52.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:52.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:52.525 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:52.525 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:52.530 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:52 smithi042 ceph-mon[113926]: pgmap v1037: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 751 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16201/36093 objects degraded (44.887%); 2942/36093 objects misplaced (8.151%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:55:52.530 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:52 smithi042 ceph-mon[108570]: pgmap v1037: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 751 MiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 16201/36093 objects degraded (44.887%); 2942/36093 objects misplaced (8.151%); 3.0 MiB/s, 47 objects/s recovering 2024-09-13T14:55:52.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:52.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.592 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:52.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:52.693 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:52.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:52.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:52.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:52.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:52.856 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:52.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:52.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:52.996 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:52.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:53.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:53.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:53.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:53.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.270 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:53.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:53.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:55:53.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:53.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:53.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:53.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:53.594 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:53.644 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.670 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:53.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:53.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:53.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:53.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:53.898 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:53.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:53.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:53.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.028 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:54.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:54.156 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:54.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:54.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:54.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:54.362 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:54 smithi159 ceph-mon[102519]: pgmap v1038: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 625 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 16258/30054 objects degraded (54.096%); 2582/30054 objects misplaced (8.591%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:54.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:55:54.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:54.578 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:54 smithi042 ceph-mon[108570]: pgmap v1038: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 625 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 16258/30054 objects degraded (54.096%); 2582/30054 objects misplaced (8.591%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:54.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:54 smithi042 ceph-mon[113926]: pgmap v1038: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 625 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 16258/30054 objects degraded (54.096%); 2582/30054 objects misplaced (8.591%); 2.9 MiB/s, 47 objects/s recovering 2024-09-13T14:55:54.680 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:54.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:54.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:54.897 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:54.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:54.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:55:54.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.067 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:55.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:55.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:55.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:55.459 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:55.561 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:55.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:55.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:55.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:55.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:55.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:55.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:55.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.860 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:55.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:55.961 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:55.986 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:55.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:55.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:56.058 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:55:56.059 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:55:56.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:56.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:56.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:56.254 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:56 smithi042 ceph-mon[108570]: pgmap v1039: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 555 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16436/26688 objects degraded (61.586%); 2085/26688 objects misplaced (7.812%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16436/26688 objects degraded (61.586%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:56 smithi042 ceph-mon[113926]: pgmap v1039: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 555 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16436/26688 objects degraded (61.586%); 2085/26688 objects misplaced (7.812%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16436/26688 objects degraded (61.586%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:56.355 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:56.405 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:56.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:56.507 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:56.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:56.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:56.634 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:56.727 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:56 smithi159 ceph-mon[102519]: pgmap v1039: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 555 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16436/26688 objects degraded (61.586%); 2085/26688 objects misplaced (7.812%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:56.728 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16436/26688 objects degraded (61.586%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:55:56.775 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:56.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:56.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:55:56.932 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1039,"stamp":"2024-09-13T14:55:54.484307+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":581834749,"num_objects":8896,"num_object_clones":0,"num_object_copies":26688,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16436,"num_objects_misplaced":2085,"num_objects_unfound":0,"num_objects_dirty":8896,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":229202,"num_write_kb":7616226,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5163,"num_bytes_recovered":335091208,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111632,"ondisk_log_size":111632,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6838060,"kb_used_data":1644464,"kb_used_omap":27,"kb_used_meta":5193444,"kb_avail":649378004,"statfs":{"total":671965249536,"available":664963076096,"internally_reserved":0,"allocated":1683931136,"data_stored":1672669276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5318087627},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-399835136,"num_objects":-6101,"num_object_clones":0,"num_object_copies":-18303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":362,"num_objects_misplaced":-1608,"num_objects_unfound":0,"num_objects_dirty":-6101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6101,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":456,"num_bytes_recovered":29884416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006759"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":411,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266102+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:55:48.266102+0000","last_peered":"2024-09-13T14:55:48.266102+0000","last_clean":"2024-09-13T14:55:48.266102+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:55:48.266102+0000","last_undegraded":"2024-09-13T14:55:48.266102+0000","last_fullsized":"2024-09-13T14:55:48.266102+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":78,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.260723+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:55:48.260723+0000","last_peered":"2024-09-13T14:55:48.260723+0000","last_clean":"2024-09-13T14:55:48.260723+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:55:48.260723+0000","last_undegraded":"2024-09-13T14:55:48.260723+0000","last_fullsized":"2024-09-13T14:55:48.260723+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":262,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612244+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:55:33.612244+0000","last_peered":"2024-09-13T14:55:33.612244+0000","last_clean":"2024-09-13T14:55:33.612244+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:55:33.612244+0000","last_undegraded":"2024-09-13T14:55:33.612244+0000","last_fullsized":"2024-09-13T14:55:33.612244+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":103,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.574428+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:55:48.574428+0000","last_peered":"2024-09-13T14:55:48.574428+0000","last_clean":"2024-09-13T14:55:48.574428+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:55:48.574428+0000","last_undegraded":"2024-09-13T14:55:48.574428+0000","last_fullsized":"2024-09-13T14:55:48.574428+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.579273+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:55:48.579273+0000","last_peered":"2024-09-13T14:55:48.579273+0000","last_clean":"2024-09-13T14:55:48.579273+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:55:48.579273+0000","last_undegraded":"2024-09-13T14:55:48.579273+0000","last_fullsized":"2024-09-13T14:55:48.579273+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":38,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.615116+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:55:33.615116+0000","last_peered":"2024-09-13T14:55:33.615116+0000","last_clean":"2024-09-13T14:55:33.615116+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:55:33.615116+0000","last_undegraded":"2024-09-13T14:55:33.615116+0000","last_fullsized":"2024-09-13T14:55:33.615116+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":437,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266743+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:55:48.266743+0000","last_peered":"2024-09-13T14:55:48.266743+0000","last_clean":"2024-09-13T14:55:48.266743+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:55:48.266743+0000","last_undegraded":"2024-09-13T14:55:48.266743+0000","last_fullsized":"2024-09-13T14:55:48.266743+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":349,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:41.564361+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:55:41.564361+0000","last_peered":"2024-09-13T14:55:41.564361+0000","last_clean":"2024-09-13T14:55:41.564361+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:55:41.564361+0000","last_undegraded":"2024-09-13T14:55:41.564361+0000","last_fullsized":"2024-09-13T14:55:41.564361+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"294'14202","reported_seq":17080,"reported_epoch":294,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:51.672029+0000","last_change":"2024-09-13T14:54:59.247898+0000","last_active":"2024-09-13T14:55:51.672029+0000","last_peered":"2024-09-13T14:55:51.672029+0000","last_clean":"2024-09-13T14:53:09.541274+0000","last_became_active":"2024-09-13T14:53:11.740035+0000","last_became_peered":"2024-09-13T14:53:11.740035+0000","last_unstale":"2024-09-13T14:55:51.672029+0000","last_undegraded":"2024-09-13T14:53:11.454953+0000","last_fullsized":"2024-09-13T14:53:11.416309+0000","mapping_epoch":271,"log_start":"265'5800","ondisk_log_start":"265'5800","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8402,"log_dups_size":0,"ondisk_log_size":8402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:21.376208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4914,"num_objects_misplaced":603,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14202,"num_write_kb":473792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1404,"num_bytes_recovered":92012544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":603},{"shards":"1,5","objects":-4913}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":28,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.569345+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:55:48.569345+0000","last_peered":"2024-09-13T14:55:48.569345+0000","last_clean":"2024-09-13T14:55:48.569345+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:55:48.569345+0000","last_undegraded":"2024-09-13T14:55:48.569345+0000","last_fullsized":"2024-09-13T14:55:48.569345+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614697+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:55:33.614697+0000","last_peered":"2024-09-13T14:55:33.614697+0000","last_clean":"2024-09-13T14:55:33.614697+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:55:33.614697+0000","last_undegraded":"2024-09-13T14:55:33.614697+0000","last_fullsized":"2024-09-13T14:55:33.614697+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"294'14573","reported_seq":14680,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:53.571110+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:55:53.571110+0000","last_peered":"2024-09-13T14:55:53.571110+0000","last_clean":"2024-09-13T14:55:53.571110+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:55:53.571110+0000","last_undegraded":"2024-09-13T14:55:53.571110+0000","last_fullsized":"2024-09-13T14:55:53.571110+0000","mapping_epoch":248,"log_start":"268'7568","ondisk_log_start":"268'7568","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":7005,"log_dups_size":0,"ondisk_log_size":7005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:35.987841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34537472,"num_objects":527,"num_object_clones":0,"num_object_copies":1581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14573,"num_write_kb":483200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":87,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265087+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:55:48.265087+0000","last_peered":"2024-09-13T14:55:48.265087+0000","last_clean":"2024-09-13T14:55:48.265087+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:55:48.265087+0000","last_undegraded":"2024-09-13T14:55:48.265087+0000","last_fullsized":"2024-09-13T14:55:48.265087+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":308,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.260918+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:55:48.260918+0000","last_peered":"2024-09-13T14:55:48.260918+0000","last_clean":"2024-09-13T14:55:48.260918+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:55:48.260918+0000","last_undegraded":"2024-09-13T14:55:48.260918+0000","last_fullsized":"2024-09-13T14:55:48.260918+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"294'14019","reported_seq":14080,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.593220+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:55:51.593220+0000","last_peered":"2024-09-13T14:55:51.593220+0000","last_clean":"2024-09-13T14:55:51.593220+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:55:51.593220+0000","last_undegraded":"2024-09-13T14:55:51.593220+0000","last_fullsized":"2024-09-13T14:55:51.593220+0000","mapping_epoch":244,"log_start":"265'5600","ondisk_log_start":"265'5600","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8419,"log_dups_size":0,"ondisk_log_size":8419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38993920,"num_objects":595,"num_object_clones":0,"num_object_copies":1785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14019,"num_write_kb":467648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":166,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612119+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:55:33.612119+0000","last_peered":"2024-09-13T14:55:33.612119+0000","last_clean":"2024-09-13T14:55:33.612119+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:55:33.612119+0000","last_undegraded":"2024-09-13T14:55:33.612119+0000","last_fullsized":"2024-09-13T14:55:33.612119+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":265,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.563881+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:55:36.563881+0000","last_peered":"2024-09-13T14:55:36.563881+0000","last_clean":"2024-09-13T14:55:36.563881+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:55:36.563881+0000","last_undegraded":"2024-09-13T14:55:36.563881+0000","last_fullsized":"2024-09-13T14:55:36.563881+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"294'14319","reported_seq":14380,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.553001+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:55:51.553001+0000","last_peered":"2024-09-13T14:55:51.553001+0000","last_clean":"2024-09-13T14:55:51.553001+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:55:51.553001+0000","last_undegraded":"2024-09-13T14:55:51.553001+0000","last_fullsized":"2024-09-13T14:55:51.553001+0000","mapping_epoch":244,"log_start":"268'7707","ondisk_log_start":"268'7707","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14319,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":264,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612123+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:55:33.612123+0000","last_peered":"2024-09-13T14:55:33.612123+0000","last_clean":"2024-09-13T14:55:33.612123+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:55:33.612123+0000","last_undegraded":"2024-09-13T14:55:33.612123+0000","last_fullsized":"2024-09-13T14:55:33.612123+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.569432+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:55:48.569432+0000","last_peered":"2024-09-13T14:55:48.569432+0000","last_clean":"2024-09-13T14:55:48.569432+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:55:48.569432+0000","last_undegraded":"2024-09-13T14:55:48.569432+0000","last_fullsized":"2024-09-13T14:55:48.569432+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"294'14251","reported_seq":14312,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.555240+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:55:51.555240+0000","last_peered":"2024-09-13T14:55:51.555240+0000","last_clean":"2024-09-13T14:55:51.555240+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:55:51.555240+0000","last_undegraded":"2024-09-13T14:55:51.555240+0000","last_fullsized":"2024-09-13T14:55:51.555240+0000","mapping_epoch":244,"log_start":"268'7645","ondisk_log_start":"268'7645","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6606,"log_dups_size":0,"ondisk_log_size":6606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36241408,"num_objects":553,"num_object_clones":0,"num_object_copies":1659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14251,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":761,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267086+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:55:48.267086+0000","last_peered":"2024-09-13T14:55:48.267086+0000","last_clean":"2024-09-13T14:55:48.267086+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:55:48.267086+0000","last_undegraded":"2024-09-13T14:55:48.267086+0000","last_fullsized":"2024-09-13T14:55:48.267086+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":454,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265911+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:55:48.265911+0000","last_peered":"2024-09-13T14:55:48.265911+0000","last_clean":"2024-09-13T14:55:48.265911+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:55:48.265911+0000","last_undegraded":"2024-09-13T14:55:48.265911+0000","last_fullsized":"2024-09-13T14:55:48.265911+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":148,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266279+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:55:48.266279+0000","last_peered":"2024-09-13T14:55:48.266279+0000","last_clean":"2024-09-13T14:55:48.266279+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:55:48.266279+0000","last_undegraded":"2024-09-13T14:55:48.266279+0000","last_fullsized":"2024-09-13T14:55:48.266279+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"294'14051","reported_seq":14173,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.855785+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:55:51.855785+0000","last_peered":"2024-09-13T14:55:51.855785+0000","last_clean":"2024-09-13T14:55:51.855785+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:55:51.855785+0000","last_undegraded":"2024-09-13T14:55:51.855785+0000","last_fullsized":"2024-09-13T14:55:51.855785+0000","mapping_epoch":248,"log_start":"265'5700","ondisk_log_start":"265'5700","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8351,"log_dups_size":0,"ondisk_log_size":8351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:06.307356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35848192,"num_objects":547,"num_object_clones":0,"num_object_copies":1641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14051,"num_write_kb":467136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":533,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614509+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:55:33.614509+0000","last_peered":"2024-09-13T14:55:33.614509+0000","last_clean":"2024-09-13T14:55:33.614509+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:55:33.614509+0000","last_undegraded":"2024-09-13T14:55:33.614509+0000","last_fullsized":"2024-09-13T14:55:33.614509+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":304,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.563892+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:55:36.563892+0000","last_peered":"2024-09-13T14:55:36.563892+0000","last_clean":"2024-09-13T14:55:36.563892+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:55:36.563892+0000","last_undegraded":"2024-09-13T14:55:36.563892+0000","last_fullsized":"2024-09-13T14:55:36.563892+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":156,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.264021+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:55:48.264021+0000","last_peered":"2024-09-13T14:55:48.264021+0000","last_clean":"2024-09-13T14:55:48.264021+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:55:48.264021+0000","last_undegraded":"2024-09-13T14:55:48.264021+0000","last_fullsized":"2024-09-13T14:55:48.264021+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"294'14062","reported_seq":14132,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.507554+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:55:51.507554+0000","last_peered":"2024-09-13T14:55:51.507554+0000","last_clean":"2024-09-13T14:55:51.507554+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:55:51.507554+0000","last_undegraded":"2024-09-13T14:55:51.507554+0000","last_fullsized":"2024-09-13T14:55:51.507554+0000","mapping_epoch":248,"log_start":"268'7451","ondisk_log_start":"268'7451","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39583744,"num_objects":604,"num_object_clones":0,"num_object_copies":1812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14062,"num_write_kb":469312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":142,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.260632+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:55:48.260632+0000","last_peered":"2024-09-13T14:55:48.260632+0000","last_clean":"2024-09-13T14:55:48.260632+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:55:48.260632+0000","last_undegraded":"2024-09-13T14:55:48.260632+0000","last_fullsized":"2024-09-13T14:55:48.260632+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614350+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:55:33.614350+0000","last_peered":"2024-09-13T14:55:33.614350+0000","last_clean":"2024-09-13T14:55:33.614350+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:55:33.614350+0000","last_undegraded":"2024-09-13T14:55:33.614350+0000","last_fullsized":"2024-09-13T14:55:33.614350+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"294'14534","reported_seq":14649,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:54.353230+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:55:54.353230+0000","last_peered":"2024-09-13T14:55:54.353230+0000","last_clean":"2024-09-13T14:55:54.353230+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:55:54.353230+0000","last_undegraded":"2024-09-13T14:55:54.353230+0000","last_fullsized":"2024-09-13T14:55:54.353230+0000","mapping_epoch":244,"log_start":"274'8452","ondisk_log_start":"274'8452","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6082,"log_dups_size":0,"ondisk_log_size":6082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:34.509903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14534,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":533,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.611839+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:55:33.611839+0000","last_peered":"2024-09-13T14:55:33.611839+0000","last_clean":"2024-09-13T14:55:33.611839+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:55:33.611839+0000","last_undegraded":"2024-09-13T14:55:33.611839+0000","last_fullsized":"2024-09-13T14:55:33.611839+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":414,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265773+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:55:48.265773+0000","last_peered":"2024-09-13T14:55:48.265773+0000","last_clean":"2024-09-13T14:55:48.265773+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:55:48.265773+0000","last_undegraded":"2024-09-13T14:55:48.265773+0000","last_fullsized":"2024-09-13T14:55:48.265773+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":419,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263672+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:55:48.263672+0000","last_peered":"2024-09-13T14:55:48.263672+0000","last_clean":"2024-09-13T14:55:48.263672+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:55:48.263672+0000","last_undegraded":"2024-09-13T14:55:48.263672+0000","last_fullsized":"2024-09-13T14:55:48.263672+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"294'14512","reported_seq":14627,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.399079+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:55:51.399079+0000","last_peered":"2024-09-13T14:55:51.399079+0000","last_clean":"2024-09-13T14:55:51.399079+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:55:51.399079+0000","last_undegraded":"2024-09-13T14:55:51.399079+0000","last_fullsized":"2024-09-13T14:55:51.399079+0000","mapping_epoch":244,"log_start":"274'8246","ondisk_log_start":"274'8246","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6266,"log_dups_size":0,"ondisk_log_size":6266,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:34.062885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41549824,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14512,"num_write_kb":484672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":488,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.568078+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:55:48.568078+0000","last_peered":"2024-09-13T14:55:48.568078+0000","last_clean":"2024-09-13T14:55:48.568078+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:55:48.568078+0000","last_undegraded":"2024-09-13T14:55:48.568078+0000","last_fullsized":"2024-09-13T14:55:48.568078+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":279,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612018+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:55:33.612018+0000","last_peered":"2024-09-13T14:55:33.612018+0000","last_clean":"2024-09-13T14:55:33.612018+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:55:33.612018+0000","last_undegraded":"2024-09-13T14:55:33.612018+0000","last_fullsized":"2024-09-13T14:55:33.612018+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":399,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269627+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:55:48.269627+0000","last_peered":"2024-09-13T14:55:48.269627+0000","last_clean":"2024-09-13T14:55:48.269627+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:55:48.269627+0000","last_undegraded":"2024-09-13T14:55:48.269627+0000","last_fullsized":"2024-09-13T14:55:48.269627+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614687+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:55:33.614687+0000","last_peered":"2024-09-13T14:55:33.614687+0000","last_clean":"2024-09-13T14:55:33.614687+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:55:33.614687+0000","last_undegraded":"2024-09-13T14:55:33.614687+0000","last_fullsized":"2024-09-13T14:55:33.614687+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"294'14247","reported_seq":16744,"reported_epoch":294,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:51.417993+0000","last_change":"2024-09-13T14:55:00.348097+0000","last_active":"2024-09-13T14:55:51.417993+0000","last_peered":"2024-09-13T14:55:51.417993+0000","last_clean":"2024-09-13T14:54:58.189452+0000","last_became_active":"2024-09-13T14:55:00.316243+0000","last_became_peered":"2024-09-13T14:55:00.316243+0000","last_unstale":"2024-09-13T14:55:51.417993+0000","last_undegraded":"2024-09-13T14:55:51.417993+0000","last_fullsized":"2024-09-13T14:55:51.417993+0000","mapping_epoch":288,"log_start":"270'7951","ondisk_log_start":"270'7951","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38731776,"num_objects":591,"num_object_clones":0,"num_object_copies":1773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":591,"num_objects_unfound":0,"num_objects_dirty":591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14247,"num_write_kb":474816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,1],"avail_no_missing":["3","1","6"],"object_location_counts":[{"shards":"1,3,6","objects":591}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":683,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.264550+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:55:48.264550+0000","last_peered":"2024-09-13T14:55:48.264550+0000","last_clean":"2024-09-13T14:55:48.264550+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:55:48.264550+0000","last_undegraded":"2024-09-13T14:55:48.264550+0000","last_fullsized":"2024-09-13T14:55:48.264550+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":400,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.262720+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:55:48.262720+0000","last_peered":"2024-09-13T14:55:48.262720+0000","last_clean":"2024-09-13T14:55:48.262720+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:55:48.262720+0000","last_undegraded":"2024-09-13T14:55:48.262720+0000","last_fullsized":"2024-09-13T14:55:48.262720+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":111,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.569266+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:55:48.569266+0000","last_peered":"2024-09-13T14:55:48.569266+0000","last_clean":"2024-09-13T14:55:48.569266+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:55:48.569266+0000","last_undegraded":"2024-09-13T14:55:48.569266+0000","last_fullsized":"2024-09-13T14:55:48.569266+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.570763+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:55:48.570763+0000","last_peered":"2024-09-13T14:55:48.570763+0000","last_clean":"2024-09-13T14:55:48.570763+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:55:48.570763+0000","last_undegraded":"2024-09-13T14:55:48.570763+0000","last_fullsized":"2024-09-13T14:55:48.570763+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"294'14176","reported_seq":14291,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.379955+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:55:51.379955+0000","last_peered":"2024-09-13T14:55:51.379955+0000","last_clean":"2024-09-13T14:55:51.379955+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:55:51.379955+0000","last_undegraded":"2024-09-13T14:55:51.379955+0000","last_fullsized":"2024-09-13T14:55:51.379955+0000","mapping_epoch":244,"log_start":"269'7832","ondisk_log_start":"269'7832","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:23.121414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14176,"num_write_kb":471744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":601,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.574457+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:55:48.574457+0000","last_peered":"2024-09-13T14:55:48.574457+0000","last_clean":"2024-09-13T14:55:48.574457+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:55:48.574457+0000","last_undegraded":"2024-09-13T14:55:48.574457+0000","last_fullsized":"2024-09-13T14:55:48.574457+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":451,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.260802+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:55:48.260802+0000","last_peered":"2024-09-13T14:55:48.260802+0000","last_clean":"2024-09-13T14:55:48.260802+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:55:48.260802+0000","last_undegraded":"2024-09-13T14:55:48.260802+0000","last_fullsized":"2024-09-13T14:55:48.260802+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":411,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.262994+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:55:48.262994+0000","last_peered":"2024-09-13T14:55:48.262994+0000","last_clean":"2024-09-13T14:55:48.262994+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:55:48.262994+0000","last_undegraded":"2024-09-13T14:55:48.262994+0000","last_fullsized":"2024-09-13T14:55:48.262994+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":462,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263528+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:55:48.263528+0000","last_peered":"2024-09-13T14:55:48.263528+0000","last_clean":"2024-09-13T14:55:48.263528+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:55:48.263528+0000","last_undegraded":"2024-09-13T14:55:48.263528+0000","last_fullsized":"2024-09-13T14:55:48.263528+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"294'14360","reported_seq":14482,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.325870+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:55:51.325870+0000","last_peered":"2024-09-13T14:55:51.325870+0000","last_clean":"2024-09-13T14:55:51.325870+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:55:51.325870+0000","last_undegraded":"2024-09-13T14:55:51.325870+0000","last_fullsized":"2024-09-13T14:55:51.325870+0000","mapping_epoch":248,"log_start":"270'8015","ondisk_log_start":"270'8015","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6345,"log_dups_size":0,"ondisk_log_size":6345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:15:37.408790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14360,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":568,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.611787+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:55:33.611787+0000","last_peered":"2024-09-13T14:55:33.611787+0000","last_clean":"2024-09-13T14:55:33.611787+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:55:33.611787+0000","last_undegraded":"2024-09-13T14:55:33.611787+0000","last_fullsized":"2024-09-13T14:55:33.611787+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":280,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612009+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:55:33.612009+0000","last_peered":"2024-09-13T14:55:33.612009+0000","last_clean":"2024-09-13T14:55:33.612009+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:55:33.612009+0000","last_undegraded":"2024-09-13T14:55:33.612009+0000","last_fullsized":"2024-09-13T14:55:33.612009+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":308,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.568275+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:55:48.568275+0000","last_peered":"2024-09-13T14:55:48.568275+0000","last_clean":"2024-09-13T14:55:48.568275+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:55:48.568275+0000","last_undegraded":"2024-09-13T14:55:48.568275+0000","last_fullsized":"2024-09-13T14:55:48.568275+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":424,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263852+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:55:48.263852+0000","last_peered":"2024-09-13T14:55:48.263852+0000","last_clean":"2024-09-13T14:55:48.263852+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:55:48.263852+0000","last_undegraded":"2024-09-13T14:55:48.263852+0000","last_fullsized":"2024-09-13T14:55:48.263852+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"294'14321","reported_seq":17580,"reported_epoch":294,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:51.824587+0000","last_change":"2024-09-13T14:54:59.252648+0000","last_active":"2024-09-13T14:55:51.824587+0000","last_peered":"2024-09-13T14:55:51.824587+0000","last_clean":"2024-09-13T14:53:09.553559+0000","last_became_active":"2024-09-13T14:53:11.733490+0000","last_became_peered":"2024-09-13T14:53:11.733490+0000","last_unstale":"2024-09-13T14:55:51.824587+0000","last_undegraded":"2024-09-13T14:53:11.458865+0000","last_fullsized":"2024-09-13T14:53:11.413105+0000","mapping_epoch":271,"log_start":"265'5900","ondisk_log_start":"265'5900","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8421,"log_dups_size":0,"ondisk_log_size":8421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41680920,"num_objects":637,"num_object_clones":0,"num_object_copies":1911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":637,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14321,"num_write_kb":478593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1569,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":637},{"shards":"2,6","objects":-4739}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":541,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614500+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:55:33.614500+0000","last_peered":"2024-09-13T14:55:33.614500+0000","last_clean":"2024-09-13T14:55:33.614500+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:55:33.614500+0000","last_undegraded":"2024-09-13T14:55:33.614500+0000","last_fullsized":"2024-09-13T14:55:33.614500+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":280,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612336+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:55:33.612336+0000","last_peered":"2024-09-13T14:55:33.612336+0000","last_clean":"2024-09-13T14:55:33.612336+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:55:33.612336+0000","last_undegraded":"2024-09-13T14:55:33.612336+0000","last_fullsized":"2024-09-13T14:55:33.612336+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":476,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.264055+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:55:48.264055+0000","last_peered":"2024-09-13T14:55:48.264055+0000","last_clean":"2024-09-13T14:55:48.264055+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:55:48.264055+0000","last_undegraded":"2024-09-13T14:55:48.264055+0000","last_fullsized":"2024-09-13T14:55:48.264055+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":339,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.262838+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:55:48.262838+0000","last_peered":"2024-09-13T14:55:48.262838+0000","last_clean":"2024-09-13T14:55:48.262838+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:55:48.262838+0000","last_undegraded":"2024-09-13T14:55:48.262838+0000","last_fullsized":"2024-09-13T14:55:48.262838+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"294'14471","reported_seq":14731,"reported_epoch":294,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:55:54.396414+0000","last_change":"2024-09-13T14:53:11.783352+0000","last_active":"2024-09-13T14:55:54.396414+0000","last_peered":"2024-09-13T14:55:54.396414+0000","last_clean":"2024-09-13T14:53:09.524137+0000","last_became_active":"2024-09-13T14:53:11.519805+0000","last_became_peered":"2024-09-13T14:53:11.519805+0000","last_unstale":"2024-09-13T14:55:54.396414+0000","last_undegraded":"2024-09-13T14:55:54.396414+0000","last_fullsized":"2024-09-13T14:55:54.396414+0000","mapping_epoch":271,"log_start":"274'8421","ondisk_log_start":"274'8421","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29294592,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":442,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14471,"num_write_kb":477376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4521984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":447}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":575,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614360+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:55:33.614360+0000","last_peered":"2024-09-13T14:55:33.614360+0000","last_clean":"2024-09-13T14:55:33.614360+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:55:33.614360+0000","last_undegraded":"2024-09-13T14:55:33.614360+0000","last_fullsized":"2024-09-13T14:55:33.614360+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":411,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263957+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:55:48.263957+0000","last_peered":"2024-09-13T14:55:48.263957+0000","last_clean":"2024-09-13T14:55:48.263957+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:55:48.263957+0000","last_undegraded":"2024-09-13T14:55:48.263957+0000","last_fullsized":"2024-09-13T14:55:48.263957+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":111,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.568522+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:55:48.568522+0000","last_peered":"2024-09-13T14:55:48.568522+0000","last_clean":"2024-09-13T14:55:48.568522+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:55:48.568522+0000","last_undegraded":"2024-09-13T14:55:48.568522+0000","last_fullsized":"2024-09-13T14:55:48.568522+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.569321+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:55:48.569321+0000","last_peered":"2024-09-13T14:55:48.569321+0000","last_clean":"2024-09-13T14:55:48.569321+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:55:48.569321+0000","last_undegraded":"2024-09-13T14:55:48.569321+0000","last_fullsized":"2024-09-13T14:55:48.569321+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"294'14517","reported_seq":14632,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:54.434860+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:55:54.434860+0000","last_peered":"2024-09-13T14:55:54.434860+0000","last_clean":"2024-09-13T14:55:54.434860+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:55:54.434860+0000","last_undegraded":"2024-09-13T14:55:54.434860+0000","last_fullsized":"2024-09-13T14:55:54.434860+0000","mapping_epoch":244,"log_start":"274'8466","ondisk_log_start":"274'8466","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:07.929154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14517,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":432,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.261057+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:55:48.261057+0000","last_peered":"2024-09-13T14:55:48.261057+0000","last_clean":"2024-09-13T14:55:48.261057+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:55:48.261057+0000","last_undegraded":"2024-09-13T14:55:48.261057+0000","last_fullsized":"2024-09-13T14:55:48.261057+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":422,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266831+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:55:48.266831+0000","last_peered":"2024-09-13T14:55:48.266831+0000","last_clean":"2024-09-13T14:55:48.266831+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:55:48.266831+0000","last_undegraded":"2024-09-13T14:55:48.266831+0000","last_fullsized":"2024-09-13T14:55:48.266831+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":419,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265171+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:55:48.265171+0000","last_peered":"2024-09-13T14:55:48.265171+0000","last_clean":"2024-09-13T14:55:48.265171+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:55:48.265171+0000","last_undegraded":"2024-09-13T14:55:48.265171+0000","last_fullsized":"2024-09-13T14:55:48.265171+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":425,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265613+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:55:48.265613+0000","last_peered":"2024-09-13T14:55:48.265613+0000","last_clean":"2024-09-13T14:55:48.265613+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:55:48.265613+0000","last_undegraded":"2024-09-13T14:55:48.265613+0000","last_fullsized":"2024-09-13T14:55:48.265613+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"294'14523","reported_seq":16399,"reported_epoch":294,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:55:54.420134+0000","last_change":"2024-09-13T14:54:59.249721+0000","last_active":"2024-09-13T14:55:54.420134+0000","last_peered":"2024-09-13T14:55:54.420134+0000","last_clean":"2024-09-13T14:53:09.526490+0000","last_became_active":"2024-09-13T14:53:11.753975+0000","last_became_peered":"2024-09-13T14:53:11.753975+0000","last_unstale":"2024-09-13T14:55:54.420134+0000","last_undegraded":"2024-09-13T14:53:11.475108+0000","last_fullsized":"2024-09-13T14:53:11.413061+0000","mapping_epoch":271,"log_start":"274'8450","ondisk_log_start":"274'8450","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5612,"num_objects_misplaced":449,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14523,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":58458112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":449},{"shards":"1,7","objects":-5612}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":672,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265958+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:55:48.265958+0000","last_peered":"2024-09-13T14:55:48.265958+0000","last_clean":"2024-09-13T14:55:48.265958+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:55:48.265958+0000","last_undegraded":"2024-09-13T14:55:48.265958+0000","last_fullsized":"2024-09-13T14:55:48.265958+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":28,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.579149+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:55:48.579149+0000","last_peered":"2024-09-13T14:55:48.579149+0000","last_clean":"2024-09-13T14:55:48.579149+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:55:48.579149+0000","last_undegraded":"2024-09-13T14:55:48.579149+0000","last_fullsized":"2024-09-13T14:55:48.579149+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":278,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614657+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:55:33.614657+0000","last_peered":"2024-09-13T14:55:33.614657+0000","last_clean":"2024-09-13T14:55:33.614657+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:55:33.614657+0000","last_undegraded":"2024-09-13T14:55:33.614657+0000","last_fullsized":"2024-09-13T14:55:33.614657+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.611858+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:55:33.611858+0000","last_peered":"2024-09-13T14:55:33.611858+0000","last_clean":"2024-09-13T14:55:33.611858+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:55:33.611858+0000","last_undegraded":"2024-09-13T14:55:33.611858+0000","last_fullsized":"2024-09-13T14:55:33.611858+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":99,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614545+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:55:33.614545+0000","last_peered":"2024-09-13T14:55:33.614545+0000","last_clean":"2024-09-13T14:55:33.614545+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:55:33.614545+0000","last_undegraded":"2024-09-13T14:55:33.614545+0000","last_fullsized":"2024-09-13T14:55:33.614545+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":97,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614822+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:55:33.614822+0000","last_peered":"2024-09-13T14:55:33.614822+0000","last_clean":"2024-09-13T14:55:33.614822+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:55:33.614822+0000","last_undegraded":"2024-09-13T14:55:33.614822+0000","last_fullsized":"2024-09-13T14:55:33.614822+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":332,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263132+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:55:48.263132+0000","last_peered":"2024-09-13T14:55:48.263132+0000","last_clean":"2024-09-13T14:55:48.263132+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:55:48.263132+0000","last_undegraded":"2024-09-13T14:55:48.263132+0000","last_fullsized":"2024-09-13T14:55:48.263132+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":370,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.261224+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:55:48.261224+0000","last_peered":"2024-09-13T14:55:48.261224+0000","last_clean":"2024-09-13T14:55:48.261224+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:55:48.261224+0000","last_undegraded":"2024-09-13T14:55:48.261224+0000","last_fullsized":"2024-09-13T14:55:48.261224+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.612384+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:55:33.612384+0000","last_peered":"2024-09-13T14:55:33.612384+0000","last_clean":"2024-09-13T14:55:33.612384+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:55:33.612384+0000","last_undegraded":"2024-09-13T14:55:33.612384+0000","last_fullsized":"2024-09-13T14:55:33.612384+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":398,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267171+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:55:48.267171+0000","last_peered":"2024-09-13T14:55:48.267171+0000","last_clean":"2024-09-13T14:55:48.267171+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:55:48.267171+0000","last_undegraded":"2024-09-13T14:55:48.267171+0000","last_fullsized":"2024-09-13T14:55:48.267171+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":103,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.570084+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:55:48.570084+0000","last_peered":"2024-09-13T14:55:48.570084+0000","last_clean":"2024-09-13T14:55:48.570084+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:55:48.570084+0000","last_undegraded":"2024-09-13T14:55:48.570084+0000","last_fullsized":"2024-09-13T14:55:48.570084+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":411,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265587+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:55:48.265587+0000","last_peered":"2024-09-13T14:55:48.265587+0000","last_clean":"2024-09-13T14:55:48.265587+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:55:48.265587+0000","last_undegraded":"2024-09-13T14:55:48.265587+0000","last_fullsized":"2024-09-13T14:55:48.265587+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":265,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:33.614587+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:55:33.614587+0000","last_peered":"2024-09-13T14:55:33.614587+0000","last_clean":"2024-09-13T14:55:33.614587+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:55:33.614587+0000","last_undegraded":"2024-09-13T14:55:33.614587+0000","last_fullsized":"2024-09-13T14:55:33.614587+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":455,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265677+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:55:48.265677+0000","last_peered":"2024-09-13T14:55:48.265677+0000","last_clean":"2024-09-13T14:55:48.265677+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:55:48.265677+0000","last_undegraded":"2024-09-13T14:55:48.265677+0000","last_fullsized":"2024-09-13T14:55:48.265677+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":104,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.579386+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:55:48.579386+0000","last_peered":"2024-09-13T14:55:48.579386+0000","last_clean":"2024-09-13T14:55:48.579386+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:55:48.579386+0000","last_undegraded":"2024-09-13T14:55:48.579386+0000","last_fullsized":"2024-09-13T14:55:48.579386+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":417,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.262833+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:55:48.262833+0000","last_peered":"2024-09-13T14:55:48.262833+0000","last_clean":"2024-09-13T14:55:48.262833+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:55:48.262833+0000","last_undegraded":"2024-09-13T14:55:48.262833+0000","last_fullsized":"2024-09-13T14:55:48.262833+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":418,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266927+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:55:48.266927+0000","last_peered":"2024-09-13T14:55:48.266927+0000","last_clean":"2024-09-13T14:55:48.266927+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:55:48.266927+0000","last_undegraded":"2024-09-13T14:55:48.266927+0000","last_fullsized":"2024-09-13T14:55:48.266927+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":76,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.263299+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:55:48.263299+0000","last_peered":"2024-09-13T14:55:48.263299+0000","last_clean":"2024-09-13T14:55:48.263299+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:55:48.263299+0000","last_undegraded":"2024-09-13T14:55:48.263299+0000","last_fullsized":"2024-09-13T14:55:48.263299+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":437,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.261162+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:55:48.261162+0000","last_peered":"2024-09-13T14:55:48.261162+0000","last_clean":"2024-09-13T14:55:48.261162+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:55:48.261162+0000","last_undegraded":"2024-09-13T14:55:48.261162+0000","last_fullsized":"2024-09-13T14:55:48.261162+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":24,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.568179+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:55:48.568179+0000","last_peered":"2024-09-13T14:55:48.568179+0000","last_clean":"2024-09-13T14:55:48.568179+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:55:48.568179+0000","last_undegraded":"2024-09-13T14:55:48.568179+0000","last_fullsized":"2024-09-13T14:55:48.568179+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.265878+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:55:48.265878+0000","last_peered":"2024-09-13T14:55:48.265878+0000","last_clean":"2024-09-13T14:55:48.265878+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:55:48.265878+0000","last_undegraded":"2024-09-13T14:55:48.265878+0000","last_fullsized":"2024-09-13T14:55:48.265878+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":278,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269434+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:55:48.269434+0000","last_peered":"2024-09-13T14:55:48.269434+0000","last_clean":"2024-09-13T14:55:48.269434+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:55:48.269434+0000","last_undegraded":"2024-09-13T14:55:48.269434+0000","last_fullsized":"2024-09-13T14:55:48.269434+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269840+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:55:48.269840+0000","last_peered":"2024-09-13T14:55:48.269840+0000","last_clean":"2024-09-13T14:55:48.269840+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:55:48.269840+0000","last_undegraded":"2024-09-13T14:55:48.269840+0000","last_fullsized":"2024-09-13T14:55:48.269840+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":263,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269616+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:55:48.269616+0000","last_peered":"2024-09-13T14:55:48.269616+0000","last_clean":"2024-09-13T14:55:48.269616+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:55:48.269616+0000","last_undegraded":"2024-09-13T14:55:48.269616+0000","last_fullsized":"2024-09-13T14:55:48.269616+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":457,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267567+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:55:48.267567+0000","last_peered":"2024-09-13T14:55:48.267567+0000","last_clean":"2024-09-13T14:55:48.267567+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:55:48.267567+0000","last_undegraded":"2024-09-13T14:55:48.267567+0000","last_fullsized":"2024-09-13T14:55:48.267567+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":433,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.266651+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:55:48.266651+0000","last_peered":"2024-09-13T14:55:48.266651+0000","last_clean":"2024-09-13T14:55:48.266651+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:55:48.266651+0000","last_undegraded":"2024-09-13T14:55:48.266651+0000","last_fullsized":"2024-09-13T14:55:48.266651+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":293,"state":"active+clean","last_fresh":"2024-09-13T14:55:36.563782+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:55:36.563782+0000","last_peered":"2024-09-13T14:55:36.563782+0000","last_clean":"2024-09-13T14:55:36.563782+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:55:36.563782+0000","last_undegraded":"2024-09-13T14:55:36.563782+0000","last_fullsized":"2024-09-13T14:55:36.563782+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":30,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.569232+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:55:48.569232+0000","last_peered":"2024-09-13T14:55:48.569232+0000","last_clean":"2024-09-13T14:55:48.569232+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:55:48.569232+0000","last_undegraded":"2024-09-13T14:55:48.569232+0000","last_fullsized":"2024-09-13T14:55:48.569232+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":581238808,"num_objects":8870,"num_object_clones":0,"num_object_copies":26610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16436,"num_objects_misplaced":2085,"num_objects_unfound":0,"num_objects_dirty":8870,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":229138,"num_write_kb":7616193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5116,"num_bytes_recovered":335085568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3003133952,"data_stored":3003121736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109934,"ondisk_log_size":109934,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234429,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1221352,"kb_used_data":233696,"kb_used_omap":1,"kb_used_meta":987646,"kb_avail":92523800,"statfs":{"total":95995035648,"available":94744371200,"internally_reserved":0,"allocated":239304704,"data_stored":237691214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1011349963},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496067,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212780,"kb_used_data":311268,"kb_used_omap":1,"kb_used_meta":901502,"kb_avail":92532372,"statfs":{"total":95995035648,"available":94753148928,"internally_reserved":0,"allocated":318738432,"data_stored":317120867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":923138503},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757705,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1223012,"kb_used_data":163096,"kb_used_omap":1,"kb_used_meta":1059838,"kb_avail":92522140,"statfs":{"total":95995035648,"available":94742671360,"internally_reserved":0,"allocated":167010304,"data_stored":165401030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1085274566},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019341,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650553,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860056,"kb_used_data":274704,"kb_used_omap":4,"kb_used_meta":585339,"kb_avail":92885096,"statfs":{"total":95995035648,"available":95114338304,"internally_reserved":0,"allocated":281296896,"data_stored":279695554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":599387695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879488,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691376,"kb_used_data":232936,"kb_used_omap":6,"kb_used_meta":458425,"kb_avail":93053776,"statfs":{"total":95995035648,"available":95287066624,"internally_reserved":0,"allocated":238526464,"data_stored":236905942,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":469427769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435505,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072464,"kb_used_data":286472,"kb_used_omap":5,"kb_used_meta":785978,"kb_avail":92672688,"statfs":{"total":95995035648,"available":94896832512,"internally_reserved":0,"allocated":293347328,"data_stored":291753039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":804841825},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926080,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557020,"kb_used_data":142292,"kb_used_omap":5,"kb_used_meta":414714,"kb_avail":93188132,"statfs":{"total":95995035648,"available":95424647168,"internally_reserved":0,"allocated":145707008,"data_stored":144101630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":424667306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":133038080,"data_stored":133038080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":275775488,"data_stored":275775488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":227741696,"data_stored":227737624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":274792448,"data_stored":274792448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1401556992,"data_stored":1401552920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":155713536,"data_stored":155713536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":309727232,"data_stored":309723160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":224788480,"data_stored":224788480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:55:56.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:56.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:55:57.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:57.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:57.334 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:57.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:57.373 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:57 smithi159 ceph-mon[102519]: from='client.45515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:57.417 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:55:57.417 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:55:57.442 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:55:57.467 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:55:57.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.526 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:57.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:55:57.530 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:57 smithi042 ceph-mon[113926]: from='client.45515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:57.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:57 smithi042 ceph-mon[108570]: from='client.45515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:55:57.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:57.679 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:57.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:57.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:57.936 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:57.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:57.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:58.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:58.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:58.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:58.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:58 smithi042 ceph-mon[113926]: pgmap v1040: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 504 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16436/24246 objects degraded (67.789%); 1880/24246 objects misplaced (7.754%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:58.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:58 smithi042 ceph-mon[108570]: pgmap v1040: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 504 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16436/24246 objects degraded (67.789%); 1880/24246 objects misplaced (7.754%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:58.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:55:58.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:58.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:55:58.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:55:58.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:55:58.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:58.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:58.738 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:58 smithi159 ceph-mon[102519]: pgmap v1040: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 504 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 612 op/s; 16436/24246 objects degraded (67.789%); 1880/24246 objects misplaced (7.754%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:55:58.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:55:58.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:58.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:58.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:58.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:58.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.061 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:55:59.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.157 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:59.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:55:59.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:59.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:55:59.317 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:55:59.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:59.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:55:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:59.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:55:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:59.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:55:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:55:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:55:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:55:59.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:59.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:55:59.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:55:59.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:55:59.909 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:55:59.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:55:59.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:55:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:55:59.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:55:59.997 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:55:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:55:59] "GET /metrics HTTP/1.1" 200 36480 "" "Prometheus/2.51.0" 2024-09-13T14:56:00.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:00.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:00.459 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:00.561 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:00.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[108570]: pgmap v1041: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 415 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16490/19992 objects degraded (82.483%); 1683/19992 objects misplaced (8.418%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[108570]: osdmap e295: 8 total, 8 up, 7 in 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[113926]: pgmap v1041: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 415 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16490/19992 objects degraded (82.483%); 1683/19992 objects misplaced (8.418%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:00 smithi042 ceph-mon[113926]: osdmap e295: 8 total, 8 up, 7 in 2024-09-13T14:56:00.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:00.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:00 smithi159 ceph-mon[102519]: pgmap v1041: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 415 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 16490/19992 objects degraded (82.483%); 1683/19992 objects misplaced (8.418%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:00 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/400518398' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:00 smithi159 ceph-mon[102519]: osdmap e295: 8 total, 8 up, 7 in 2024-09-13T14:56:00.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:00.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:00.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:00.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:00.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:00.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:01.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:01.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:01.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:01.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:01.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:01.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:01.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:01.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:01.408 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:01 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16490/19992 objects degraded (82.483%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:01.408 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:01 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16490/19992 objects degraded (82.483%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:01.418 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:01.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:01.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:01.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:01.533 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:56:01.533 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:01.566 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:56:01.566 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:01.581 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:01 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16490/19992 objects degraded (82.483%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:01.599 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:56:01.600 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:01.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:01.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:01.631 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:56:01.632 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:01.675 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:56:01.675 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:01.708 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:56:01.709 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:01.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:01.739 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:56:01.739 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:01.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:56:01.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:01.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:01.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:01.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:01.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:02.016 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.058 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:02.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:02.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:02.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:02.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:02.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45314.0:235529 11.3 11:c8516f20:::benchmark_data_smithi042_223320_object116523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:56:02.381446+0000", 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.003277383, 2024-09-13T14:56:02.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033001319999999999, 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45314", 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/339527939", 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 235529 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381446+0000", 2024-09-13T14:56:02.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381446+0000", 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:02.388 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381445+0000", 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381451+0000", 2024-09-13T14:56:02.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9159999999999998e-06 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381463+0000", 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2132e-05 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:02.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381468+0000", 2024-09-13T14:56:02.391 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3009999999999997e-06 2024-09-13T14:56:02.391 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.391 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.391 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:02.391 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381503+0000", 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4891000000000002e-05 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381597+0000", 2024-09-13T14:56:02.392 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.3604999999999996e-05 2024-09-13T14:56:02.393 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.393 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.393 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T14:56:02.393 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.381669+0000", 2024-09-13T14:56:02.393 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2286000000000006e-05 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:02.384403+0000", 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027332659999999998 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:02.394 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:56:02.395 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 1 2024-09-13T14:56:02.395 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:56:02.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:02.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:02.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:02.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:02 smithi042 ceph-mon[108570]: pgmap v1043: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 356 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 16581/17124 objects degraded (96.829%); 1318/17124 objects misplaced (7.697%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1091632835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:02 smithi042 ceph-mon[113926]: pgmap v1043: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 356 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 16581/17124 objects degraded (96.829%); 1318/17124 objects misplaced (7.697%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1091632835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:02.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:02.704 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:02.718 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:02 smithi159 ceph-mon[102519]: pgmap v1043: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 356 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 16581/17124 objects degraded (96.829%); 1318/17124 objects misplaced (7.697%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:56:02.719 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1091632835' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:02.743 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:02.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:02.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:56:02.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:02.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:02.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:03.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:03.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:56:03.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:03.170 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:03.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:03.272 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.307 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.378 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:03.409 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:03.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:03.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:03 smithi159 ceph-mon[102519]: pgmap v1044: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 217 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 16640/10473 objects degraded (158.885%); 903/10473 objects misplaced (8.622%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:56:03.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:03 smithi042 ceph-mon[113926]: pgmap v1044: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 217 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 16640/10473 objects degraded (158.885%); 903/10473 objects misplaced (8.622%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:56:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:03 smithi042 ceph-mon[108570]: pgmap v1044: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 217 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 16640/10473 objects degraded (158.885%); 903/10473 objects misplaced (8.622%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T14:56:03.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:03.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:03.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:03.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:03.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:03.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:03.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:04.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:04.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:04.179 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:04.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:04.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:04.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.505 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:04.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.608 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:04.632 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:04.734 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:04.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:04.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:04.915 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:04.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.016 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:05.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.039 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:05.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:05.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:05.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:05.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:05.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:05.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:05.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.617 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:05.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:05.718 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:05.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:05.837 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:05 smithi042 ceph-mon[108570]: pgmap v1045: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 154 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 16752/7464 objects degraded (224.437%); 516/7464 objects misplaced (6.913%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:05.837 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16752/7464 objects degraded (224.437%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:05.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:05 smithi042 ceph-mon[113926]: pgmap v1045: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 154 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 16752/7464 objects degraded (224.437%); 516/7464 objects misplaced (6.913%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:05.838 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16752/7464 objects degraded (224.437%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:05.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:05.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:05.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:05 smithi159 ceph-mon[102519]: pgmap v1045: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 154 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 16752/7464 objects degraded (224.437%); 516/7464 objects misplaced (6.913%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16752/7464 objects degraded (224.437%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:06.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:06.120 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:06.186 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 119003 objects 2024-09-13T14:56:06.187 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :185.402 2024-09-13T14:56:06.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.267 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.369 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:06.374 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:06.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:06.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:06.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:06.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:06.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:06.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:06.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.768 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:06.870 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:06.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:06.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:06.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:06.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:07.018 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:07.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:07.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:07.123 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:56:07.124 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:56:07.141 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:07.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:07.252 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:56:07.252 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:56:07.252 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:56:07.252 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:56:07.252 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:56:07.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:07.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:07.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:07.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:07.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:07.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.515 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:07.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:07.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:07.638 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:07.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:07.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:07.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:07.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:07 smithi042 ceph-mon[108570]: pgmap v1046: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 106 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16752/5142 objects degraded (325.788%); 337/5142 objects misplaced (6.554%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:07.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:07 smithi042 ceph-mon[108570]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:07 smithi042 ceph-mon[113926]: pgmap v1046: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 106 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16752/5142 objects degraded (325.788%); 337/5142 objects misplaced (6.554%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:07 smithi042 ceph-mon[113926]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:07.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:07.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:07.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:07.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:07 smithi159 ceph-mon[102519]: pgmap v1046: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 106 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 16752/5142 objects degraded (325.788%); 337/5142 objects misplaced (6.554%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:56:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:07 smithi159 ceph-mon[102519]: from='client.35562 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:08.013 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:08.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.051 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1046,"stamp":"2024-09-13T14:56:06.489562+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":111155197,"num_objects":1714,"num_object_clones":0,"num_object_copies":5142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16752,"num_objects_misplaced":337,"num_objects_unfound":0,"num_objects_dirty":1714,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":236384,"num_write_kb":7616226,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5633,"num_bytes_recovered":365893128,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111214,"ondisk_log_size":111214,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5654040,"kb_used_data":426016,"kb_used_omap":27,"kb_used_meta":5227940,"kb_avail":650562024,"statfs":{"total":671965249536,"available":666175512576,"internally_reserved":0,"allocated":436240384,"data_stored":424923133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5353411531},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,0,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417333248,"num_objects":-6368,"num_object_clones":0,"num_object_copies":-19104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":316,"num_objects_misplaced":-1543,"num_objects_unfound":0,"num_objects_dirty":-6368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6368,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":470,"num_bytes_recovered":30801920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004626"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":413,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319979+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:55:59.319979+0000","last_peered":"2024-09-13T14:55:59.319979+0000","last_clean":"2024-09-13T14:55:59.319979+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:55:59.319979+0000","last_undegraded":"2024-09-13T14:55:59.319979+0000","last_fullsized":"2024-09-13T14:55:59.319979+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":80,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.333560+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:55:59.333560+0000","last_peered":"2024-09-13T14:55:59.333560+0000","last_clean":"2024-09-13T14:55:59.333560+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:55:59.333560+0000","last_undegraded":"2024-09-13T14:55:59.333560+0000","last_fullsized":"2024-09-13T14:55:59.333560+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":263,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269531+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:55:48.269531+0000","last_peered":"2024-09-13T14:55:48.269531+0000","last_clean":"2024-09-13T14:55:48.269531+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:55:48.269531+0000","last_undegraded":"2024-09-13T14:55:48.269531+0000","last_fullsized":"2024-09-13T14:55:48.269531+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":106,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.343583+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:55:59.343583+0000","last_peered":"2024-09-13T14:55:59.343583+0000","last_clean":"2024-09-13T14:55:59.343583+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:55:59.343583+0000","last_undegraded":"2024-09-13T14:55:59.343583+0000","last_fullsized":"2024-09-13T14:55:59.343583+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":105,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342562+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:55:59.342562+0000","last_peered":"2024-09-13T14:55:59.342562+0000","last_clean":"2024-09-13T14:55:59.342562+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:55:59.342562+0000","last_undegraded":"2024-09-13T14:55:59.342562+0000","last_fullsized":"2024-09-13T14:55:59.342562+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":39,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267971+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:55:48.267971+0000","last_peered":"2024-09-13T14:55:48.267971+0000","last_clean":"2024-09-13T14:55:48.267971+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:55:48.267971+0000","last_undegraded":"2024-09-13T14:55:48.267971+0000","last_fullsized":"2024-09-13T14:55:48.267971+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":439,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320771+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:55:59.320771+0000","last_peered":"2024-09-13T14:55:59.320771+0000","last_clean":"2024-09-13T14:55:59.320771+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:55:59.320771+0000","last_undegraded":"2024-09-13T14:55:59.320771+0000","last_fullsized":"2024-09-13T14:55:59.320771+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":350,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267829+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:55:48.267829+0000","last_peered":"2024-09-13T14:55:48.267829+0000","last_clean":"2024-09-13T14:55:48.267829+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:55:48.267829+0000","last_undegraded":"2024-09-13T14:55:48.267829+0000","last_fullsized":"2024-09-13T14:55:48.267829+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.7","version":"295'14607","reported_seq":18070,"reported_epoch":295,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:56:01.671677+0000","last_change":"2024-09-13T14:54:59.247898+0000","last_active":"2024-09-13T14:56:01.671677+0000","last_peered":"2024-09-13T14:56:01.671677+0000","last_clean":"2024-09-13T14:53:09.541274+0000","last_became_active":"2024-09-13T14:53:11.740035+0000","last_became_peered":"2024-09-13T14:53:11.740035+0000","last_unstale":"2024-09-13T14:56:01.671677+0000","last_undegraded":"2024-09-13T14:53:11.454953+0000","last_fullsized":"2024-09-13T14:53:11.416309+0000","mapping_epoch":271,"log_start":"266'6200","ondisk_log_start":"266'6200","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8407,"log_dups_size":0,"ondisk_log_size":8407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:21.376208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5027,"num_objects_misplaced":198,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14607,"num_write_kb":473792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1696,"num_bytes_recovered":111149056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":198},{"shards":"1,5","objects":-5026}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":30,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.347513+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:55:59.347513+0000","last_peered":"2024-09-13T14:55:59.347513+0000","last_clean":"2024-09-13T14:55:59.347513+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:55:59.347513+0000","last_undegraded":"2024-09-13T14:55:59.347513+0000","last_fullsized":"2024-09-13T14:55:59.347513+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":250,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268958+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:55:48.268958+0000","last_peered":"2024-09-13T14:55:48.268958+0000","last_clean":"2024-09-13T14:55:48.268958+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:55:48.268958+0000","last_undegraded":"2024-09-13T14:55:48.268958+0000","last_fullsized":"2024-09-13T14:55:48.268958+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.6","version":"295'14985","reported_seq":15094,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:03.579640+0000","last_change":"2024-09-13T14:51:20.406297+0000","last_active":"2024-09-13T14:56:03.579640+0000","last_peered":"2024-09-13T14:56:03.579640+0000","last_clean":"2024-09-13T14:56:03.579640+0000","last_became_active":"2024-09-13T14:51:20.405939+0000","last_became_peered":"2024-09-13T14:51:20.405939+0000","last_unstale":"2024-09-13T14:56:03.579640+0000","last_undegraded":"2024-09-13T14:56:03.579640+0000","last_fullsized":"2024-09-13T14:56:03.579640+0000","mapping_epoch":248,"log_start":"269'7968","ondisk_log_start":"269'7968","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":7017,"log_dups_size":0,"ondisk_log_size":7017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:49:35.987841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7536640,"num_objects":115,"num_object_clones":0,"num_object_copies":345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14985,"num_write_kb":483200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":89,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.362458+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:55:59.362458+0000","last_peered":"2024-09-13T14:55:59.362458+0000","last_clean":"2024-09-13T14:55:59.362458+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:55:59.362458+0000","last_undegraded":"2024-09-13T14:55:59.362458+0000","last_fullsized":"2024-09-13T14:55:59.362458+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":310,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.333034+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:55:59.333034+0000","last_peered":"2024-09-13T14:55:59.333034+0000","last_clean":"2024-09-13T14:55:59.333034+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:55:59.333034+0000","last_undegraded":"2024-09-13T14:55:59.333034+0000","last_fullsized":"2024-09-13T14:55:59.333034+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.5","version":"295'14438","reported_seq":14498,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.637463+0000","last_change":"2024-09-13T14:51:17.034695+0000","last_active":"2024-09-13T14:56:01.637463+0000","last_peered":"2024-09-13T14:56:01.637463+0000","last_clean":"2024-09-13T14:56:01.637463+0000","last_became_active":"2024-09-13T14:51:17.034367+0000","last_became_peered":"2024-09-13T14:51:17.034367+0000","last_unstale":"2024-09-13T14:56:01.637463+0000","last_undegraded":"2024-09-13T14:56:01.637463+0000","last_fullsized":"2024-09-13T14:56:01.637463+0000","mapping_epoch":244,"log_start":"266'6100","ondisk_log_start":"266'6100","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8338,"log_dups_size":0,"ondisk_log_size":8338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:58:53.570482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534336,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14438,"num_write_kb":467648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":167,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269463+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:55:48.269463+0000","last_peered":"2024-09-13T14:55:48.269463+0000","last_clean":"2024-09-13T14:55:48.269463+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:55:48.269463+0000","last_undegraded":"2024-09-13T14:55:48.269463+0000","last_fullsized":"2024-09-13T14:55:48.269463+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":266,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269668+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:55:48.269668+0000","last_peered":"2024-09-13T14:55:48.269668+0000","last_clean":"2024-09-13T14:55:48.269668+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:55:48.269668+0000","last_undegraded":"2024-09-13T14:55:48.269668+0000","last_fullsized":"2024-09-13T14:55:48.269668+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.4","version":"295'14677","reported_seq":14739,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.553093+0000","last_change":"2024-09-13T14:51:17.034656+0000","last_active":"2024-09-13T14:56:01.553093+0000","last_peered":"2024-09-13T14:56:01.553093+0000","last_clean":"2024-09-13T14:56:01.553093+0000","last_became_active":"2024-09-13T14:51:17.034327+0000","last_became_peered":"2024-09-13T14:51:17.034327+0000","last_unstale":"2024-09-13T14:56:01.553093+0000","last_undegraded":"2024-09-13T14:56:01.553093+0000","last_fullsized":"2024-09-13T14:56:01.553093+0000","mapping_epoch":244,"log_start":"274'8107","ondisk_log_start":"274'8107","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6570,"log_dups_size":0,"ondisk_log_size":6570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:05.354481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12779520,"num_objects":195,"num_object_clones":0,"num_object_copies":585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14677,"num_write_kb":475904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":265,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269542+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:55:48.269542+0000","last_peered":"2024-09-13T14:55:48.269542+0000","last_clean":"2024-09-13T14:55:48.269542+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:55:48.269542+0000","last_undegraded":"2024-09-13T14:55:48.269542+0000","last_fullsized":"2024-09-13T14:55:48.269542+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342012+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:55:59.342012+0000","last_peered":"2024-09-13T14:55:59.342012+0000","last_clean":"2024-09-13T14:55:59.342012+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:55:59.342012+0000","last_undegraded":"2024-09-13T14:55:59.342012+0000","last_fullsized":"2024-09-13T14:55:59.342012+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.3","version":"295'14636","reported_seq":14698,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.522417+0000","last_change":"2024-09-13T14:51:17.034650+0000","last_active":"2024-09-13T14:56:01.522417+0000","last_peered":"2024-09-13T14:56:01.522417+0000","last_clean":"2024-09-13T14:56:01.522417+0000","last_became_active":"2024-09-13T14:51:17.034299+0000","last_became_peered":"2024-09-13T14:51:17.034299+0000","last_unstale":"2024-09-13T14:56:01.522417+0000","last_undegraded":"2024-09-13T14:56:01.522417+0000","last_fullsized":"2024-09-13T14:56:01.522417+0000","mapping_epoch":244,"log_start":"274'8045","ondisk_log_start":"274'8045","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:50:47.237022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14636,"num_write_kb":473728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":763,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319641+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:55:59.319641+0000","last_peered":"2024-09-13T14:55:59.319641+0000","last_clean":"2024-09-13T14:55:59.319641+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:55:59.319641+0000","last_undegraded":"2024-09-13T14:55:59.319641+0000","last_fullsized":"2024-09-13T14:55:59.319641+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":456,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320565+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:55:59.320565+0000","last_peered":"2024-09-13T14:55:59.320565+0000","last_clean":"2024-09-13T14:55:59.320565+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:55:59.320565+0000","last_undegraded":"2024-09-13T14:55:59.320565+0000","last_fullsized":"2024-09-13T14:55:59.320565+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320192+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:55:59.320192+0000","last_peered":"2024-09-13T14:55:59.320192+0000","last_clean":"2024-09-13T14:55:59.320192+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:55:59.320192+0000","last_undegraded":"2024-09-13T14:55:59.320192+0000","last_fullsized":"2024-09-13T14:55:59.320192+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.2","version":"295'14453","reported_seq":14577,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.803094+0000","last_change":"2024-09-13T14:51:20.408742+0000","last_active":"2024-09-13T14:56:01.803094+0000","last_peered":"2024-09-13T14:56:01.803094+0000","last_clean":"2024-09-13T14:56:01.803094+0000","last_became_active":"2024-09-13T14:51:20.408404+0000","last_became_peered":"2024-09-13T14:51:20.408404+0000","last_unstale":"2024-09-13T14:56:01.803094+0000","last_undegraded":"2024-09-13T14:56:01.803094+0000","last_fullsized":"2024-09-13T14:56:01.803094+0000","mapping_epoch":248,"log_start":"266'6100","ondisk_log_start":"266'6100","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8353,"log_dups_size":0,"ondisk_log_size":8353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:06.307356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9502720,"num_objects":145,"num_object_clones":0,"num_object_copies":435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14453,"num_write_kb":467136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":535,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:56.678406+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:55:56.678406+0000","last_peered":"2024-09-13T14:55:56.678406+0000","last_clean":"2024-09-13T14:55:56.678406+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:55:56.678406+0000","last_undegraded":"2024-09-13T14:55:56.678406+0000","last_fullsized":"2024-09-13T14:55:56.678406+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":305,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269782+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:55:48.269782+0000","last_peered":"2024-09-13T14:55:48.269782+0000","last_clean":"2024-09-13T14:55:48.269782+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:55:48.269782+0000","last_undegraded":"2024-09-13T14:55:48.269782+0000","last_fullsized":"2024-09-13T14:55:48.269782+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":158,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.338230+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:55:59.338230+0000","last_peered":"2024-09-13T14:55:59.338230+0000","last_clean":"2024-09-13T14:55:59.338230+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:55:59.338230+0000","last_undegraded":"2024-09-13T14:55:59.338230+0000","last_fullsized":"2024-09-13T14:55:59.338230+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.1","version":"295'14481","reported_seq":14552,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:01.513375+0000","last_change":"2024-09-13T14:51:20.399454+0000","last_active":"2024-09-13T14:56:01.513375+0000","last_peered":"2024-09-13T14:56:01.513375+0000","last_clean":"2024-09-13T14:56:01.513375+0000","last_became_active":"2024-09-13T14:51:20.399096+0000","last_became_peered":"2024-09-13T14:51:20.399096+0000","last_unstale":"2024-09-13T14:56:01.513375+0000","last_undegraded":"2024-09-13T14:56:01.513375+0000","last_fullsized":"2024-09-13T14:56:01.513375+0000","mapping_epoch":248,"log_start":"274'7951","ondisk_log_start":"274'7951","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:51.340617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12124160,"num_objects":185,"num_object_clones":0,"num_object_copies":555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14481,"num_write_kb":469312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":144,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.333252+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:55:59.333252+0000","last_peered":"2024-09-13T14:55:59.333252+0000","last_clean":"2024-09-13T14:55:59.333252+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:55:59.333252+0000","last_undegraded":"2024-09-13T14:55:59.333252+0000","last_fullsized":"2024-09-13T14:55:59.333252+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":127,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268175+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:55:48.268175+0000","last_peered":"2024-09-13T14:55:48.268175+0000","last_clean":"2024-09-13T14:55:48.268175+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:55:48.268175+0000","last_undegraded":"2024-09-13T14:55:48.268175+0000","last_fullsized":"2024-09-13T14:55:48.268175+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.0","version":"295'14941","reported_seq":15058,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:04.449462+0000","last_change":"2024-09-13T14:51:17.035306+0000","last_active":"2024-09-13T14:56:04.449462+0000","last_peered":"2024-09-13T14:56:04.449462+0000","last_clean":"2024-09-13T14:56:04.449462+0000","last_became_active":"2024-09-13T14:51:17.034780+0000","last_became_peered":"2024-09-13T14:51:17.034780+0000","last_unstale":"2024-09-13T14:56:04.449462+0000","last_undegraded":"2024-09-13T14:56:04.449462+0000","last_fullsized":"2024-09-13T14:56:04.449462+0000","mapping_epoch":244,"log_start":"276'8852","ondisk_log_start":"276'8852","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6089,"log_dups_size":0,"ondisk_log_size":6089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:34.509903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5701632,"num_objects":87,"num_object_clones":0,"num_object_copies":261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14941,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":535,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342716+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:55:59.342716+0000","last_peered":"2024-09-13T14:55:59.342716+0000","last_clean":"2024-09-13T14:55:59.342716+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:55:59.342716+0000","last_undegraded":"2024-09-13T14:55:59.342716+0000","last_fullsized":"2024-09-13T14:55:59.342716+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":416,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320093+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:55:59.320093+0000","last_peered":"2024-09-13T14:55:59.320093+0000","last_clean":"2024-09-13T14:55:59.320093+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:55:59.320093+0000","last_undegraded":"2024-09-13T14:55:59.320093+0000","last_fullsized":"2024-09-13T14:55:59.320093+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":421,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341379+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:55:59.341379+0000","last_peered":"2024-09-13T14:55:59.341379+0000","last_clean":"2024-09-13T14:55:59.341379+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:55:59.341379+0000","last_undegraded":"2024-09-13T14:55:59.341379+0000","last_fullsized":"2024-09-13T14:55:59.341379+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.a","version":"295'15146","reported_seq":15263,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:06.153001+0000","last_change":"2024-09-13T14:51:16.448356+0000","last_active":"2024-09-13T14:56:06.153001+0000","last_peered":"2024-09-13T14:56:06.153001+0000","last_clean":"2024-09-13T14:56:06.153001+0000","last_became_active":"2024-09-13T14:51:16.447822+0000","last_became_peered":"2024-09-13T14:51:16.447822+0000","last_unstale":"2024-09-13T14:56:06.153001+0000","last_undegraded":"2024-09-13T14:56:06.153001+0000","last_fullsized":"2024-09-13T14:56:06.153001+0000","mapping_epoch":244,"log_start":"276'8846","ondisk_log_start":"276'8846","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6300,"log_dups_size":0,"ondisk_log_size":6300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:49:34.062885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15146,"num_write_kb":484672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":490,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341118+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:55:59.341118+0000","last_peered":"2024-09-13T14:55:59.341118+0000","last_clean":"2024-09-13T14:55:59.341118+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:55:59.341118+0000","last_undegraded":"2024-09-13T14:55:59.341118+0000","last_fullsized":"2024-09-13T14:55:59.341118+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":280,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269344+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:55:48.269344+0000","last_peered":"2024-09-13T14:55:48.269344+0000","last_clean":"2024-09-13T14:55:48.269344+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:55:48.269344+0000","last_undegraded":"2024-09-13T14:55:48.269344+0000","last_fullsized":"2024-09-13T14:55:48.269344+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":401,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320717+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:55:59.320717+0000","last_peered":"2024-09-13T14:55:59.320717+0000","last_clean":"2024-09-13T14:55:59.320717+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:55:59.320717+0000","last_undegraded":"2024-09-13T14:55:59.320717+0000","last_fullsized":"2024-09-13T14:55:59.320717+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268283+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:55:48.268283+0000","last_peered":"2024-09-13T14:55:48.268283+0000","last_clean":"2024-09-13T14:55:48.268283+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:55:48.268283+0000","last_undegraded":"2024-09-13T14:55:48.268283+0000","last_fullsized":"2024-09-13T14:55:48.268283+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.9","version":"295'14838","reported_seq":17338,"reported_epoch":295,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:56:06.112349+0000","last_change":"2024-09-13T14:55:00.348097+0000","last_active":"2024-09-13T14:56:06.112349+0000","last_peered":"2024-09-13T14:56:06.112349+0000","last_clean":"2024-09-13T14:54:58.189452+0000","last_became_active":"2024-09-13T14:55:00.316243+0000","last_became_peered":"2024-09-13T14:55:00.316243+0000","last_unstale":"2024-09-13T14:56:06.112349+0000","last_undegraded":"2024-09-13T14:56:06.112349+0000","last_fullsized":"2024-09-13T14:56:06.112349+0000","mapping_epoch":288,"log_start":"275'8551","ondisk_log_start":"275'8551","created":244,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6287,"log_dups_size":0,"ondisk_log_size":6287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14838,"num_write_kb":474816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1180,"num_bytes_recovered":77332480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,1],"avail_no_missing":["3","0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":685,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.340814+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:55:59.340814+0000","last_peered":"2024-09-13T14:55:59.340814+0000","last_clean":"2024-09-13T14:55:59.340814+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:55:59.340814+0000","last_undegraded":"2024-09-13T14:55:59.340814+0000","last_fullsized":"2024-09-13T14:55:59.340814+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":402,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.336681+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:55:59.336681+0000","last_peered":"2024-09-13T14:55:59.336681+0000","last_clean":"2024-09-13T14:55:59.336681+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:55:59.336681+0000","last_undegraded":"2024-09-13T14:55:59.336681+0000","last_fullsized":"2024-09-13T14:55:59.336681+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341567+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:55:59.341567+0000","last_peered":"2024-09-13T14:55:59.341567+0000","last_clean":"2024-09-13T14:55:59.341567+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:55:59.341567+0000","last_undegraded":"2024-09-13T14:55:59.341567+0000","last_fullsized":"2024-09-13T14:55:59.341567+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":26,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341607+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:55:59.341607+0000","last_peered":"2024-09-13T14:55:59.341607+0000","last_clean":"2024-09-13T14:55:59.341607+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:55:59.341607+0000","last_undegraded":"2024-09-13T14:55:59.341607+0000","last_fullsized":"2024-09-13T14:55:59.341607+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.8","version":"295'14742","reported_seq":14859,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:06.054309+0000","last_change":"2024-09-13T14:51:16.448945+0000","last_active":"2024-09-13T14:56:06.054309+0000","last_peered":"2024-09-13T14:56:06.054309+0000","last_clean":"2024-09-13T14:56:06.054309+0000","last_became_active":"2024-09-13T14:51:16.447868+0000","last_became_peered":"2024-09-13T14:51:16.447868+0000","last_unstale":"2024-09-13T14:56:06.054309+0000","last_undegraded":"2024-09-13T14:56:06.054309+0000","last_fullsized":"2024-09-13T14:56:06.054309+0000","mapping_epoch":244,"log_start":"275'8432","ondisk_log_start":"275'8432","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6310,"log_dups_size":0,"ondisk_log_size":6310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:23.121414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14742,"num_write_kb":471744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":603,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.343629+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:55:59.343629+0000","last_peered":"2024-09-13T14:55:59.343629+0000","last_clean":"2024-09-13T14:55:59.343629+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:55:59.343629+0000","last_undegraded":"2024-09-13T14:55:59.343629+0000","last_fullsized":"2024-09-13T14:55:59.343629+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":453,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.333419+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:55:59.333419+0000","last_peered":"2024-09-13T14:55:59.333419+0000","last_clean":"2024-09-13T14:55:59.333419+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:55:59.333419+0000","last_undegraded":"2024-09-13T14:55:59.333419+0000","last_fullsized":"2024-09-13T14:55:59.333419+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":414,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.337014+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:55:59.337014+0000","last_peered":"2024-09-13T14:55:59.337014+0000","last_clean":"2024-09-13T14:55:59.337014+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:55:59.337014+0000","last_undegraded":"2024-09-13T14:55:59.337014+0000","last_fullsized":"2024-09-13T14:55:59.337014+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":464,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341216+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:55:59.341216+0000","last_peered":"2024-09-13T14:55:59.341216+0000","last_clean":"2024-09-13T14:55:59.341216+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:55:59.341216+0000","last_undegraded":"2024-09-13T14:55:59.341216+0000","last_fullsized":"2024-09-13T14:55:59.341216+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.b","version":"295'14974","reported_seq":15098,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:06.143620+0000","last_change":"2024-09-13T14:51:20.408529+0000","last_active":"2024-09-13T14:56:06.143620+0000","last_peered":"2024-09-13T14:56:06.143620+0000","last_clean":"2024-09-13T14:56:06.143620+0000","last_became_active":"2024-09-13T14:51:20.408009+0000","last_became_peered":"2024-09-13T14:51:20.408009+0000","last_unstale":"2024-09-13T14:56:06.143620+0000","last_undegraded":"2024-09-13T14:56:06.143620+0000","last_fullsized":"2024-09-13T14:56:06.143620+0000","mapping_epoch":248,"log_start":"276'8715","ondisk_log_start":"276'8715","created":244,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:15:37.408790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14974,"num_write_kb":479168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":569,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268964+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:55:48.268964+0000","last_peered":"2024-09-13T14:55:48.268964+0000","last_clean":"2024-09-13T14:55:48.268964+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:55:48.268964+0000","last_undegraded":"2024-09-13T14:55:48.268964+0000","last_fullsized":"2024-09-13T14:55:48.268964+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":281,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.270804+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:55:48.270804+0000","last_peered":"2024-09-13T14:55:48.270804+0000","last_clean":"2024-09-13T14:55:48.270804+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:55:48.270804+0000","last_undegraded":"2024-09-13T14:55:48.270804+0000","last_fullsized":"2024-09-13T14:55:48.270804+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":310,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341792+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:55:59.341792+0000","last_peered":"2024-09-13T14:55:59.341792+0000","last_clean":"2024-09-13T14:55:59.341792+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:55:59.341792+0000","last_undegraded":"2024-09-13T14:55:59.341792+0000","last_fullsized":"2024-09-13T14:55:59.341792+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":426,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.338701+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:55:59.338701+0000","last_peered":"2024-09-13T14:55:59.338701+0000","last_clean":"2024-09-13T14:55:59.338701+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:55:59.338701+0000","last_undegraded":"2024-09-13T14:55:59.338701+0000","last_fullsized":"2024-09-13T14:55:59.338701+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.d","version":"295'14751","reported_seq":18012,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:56:01.851130+0000","last_change":"2024-09-13T14:54:59.252648+0000","last_active":"2024-09-13T14:56:01.851130+0000","last_peered":"2024-09-13T14:56:01.851130+0000","last_clean":"2024-09-13T14:53:09.553559+0000","last_became_active":"2024-09-13T14:53:11.733490+0000","last_became_peered":"2024-09-13T14:53:11.733490+0000","last_unstale":"2024-09-13T14:56:01.851130+0000","last_undegraded":"2024-09-13T14:53:11.458865+0000","last_fullsized":"2024-09-13T14:53:11.413105+0000","mapping_epoch":271,"log_start":"267'6400","ondisk_log_start":"267'6400","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":8351,"log_dups_size":0,"ondisk_log_size":8351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13500440,"num_objects":207,"num_object_clones":0,"num_object_copies":621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5910,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":207,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14751,"num_write_kb":478593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1569,"num_bytes_recovered":102760448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4,6","objects":207},{"shards":"2,6","objects":-5169}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":543,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:56.567225+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:55:56.567225+0000","last_peered":"2024-09-13T14:55:56.567225+0000","last_clean":"2024-09-13T14:55:56.567225+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:55:56.567225+0000","last_undegraded":"2024-09-13T14:55:56.567225+0000","last_fullsized":"2024-09-13T14:55:56.567225+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":281,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269693+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:55:48.269693+0000","last_peered":"2024-09-13T14:55:48.269693+0000","last_clean":"2024-09-13T14:55:48.269693+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:55:48.269693+0000","last_undegraded":"2024-09-13T14:55:48.269693+0000","last_fullsized":"2024-09-13T14:55:48.269693+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":478,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.339132+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:55:59.339132+0000","last_peered":"2024-09-13T14:55:59.339132+0000","last_clean":"2024-09-13T14:55:59.339132+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:55:59.339132+0000","last_undegraded":"2024-09-13T14:55:59.339132+0000","last_fullsized":"2024-09-13T14:55:59.339132+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":342,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.336697+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:55:59.336697+0000","last_peered":"2024-09-13T14:55:59.336697+0000","last_clean":"2024-09-13T14:55:59.336697+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:55:59.336697+0000","last_undegraded":"2024-09-13T14:55:59.336697+0000","last_fullsized":"2024-09-13T14:55:59.336697+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.c","version":"295'14846","reported_seq":15108,"reported_epoch":295,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T14:56:04.467970+0000","last_change":"2024-09-13T14:53:11.783352+0000","last_active":"2024-09-13T14:56:04.467970+0000","last_peered":"2024-09-13T14:56:04.467970+0000","last_clean":"2024-09-13T14:53:09.524137+0000","last_became_active":"2024-09-13T14:53:11.519805+0000","last_became_peered":"2024-09-13T14:53:11.519805+0000","last_unstale":"2024-09-13T14:56:04.467970+0000","last_undegraded":"2024-09-13T14:56:04.467970+0000","last_fullsized":"2024-09-13T14:56:04.467970+0000","mapping_epoch":271,"log_start":"276'8821","ondisk_log_start":"276'8821","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4718592,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":71,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14846,"num_write_kb":477376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":69,"num_bytes_recovered":4521984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[{"shards":"1,2,4","objects":72}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":577,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:56.567209+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:55:56.567209+0000","last_peered":"2024-09-13T14:55:56.567209+0000","last_clean":"2024-09-13T14:55:56.567209+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:55:56.567209+0000","last_undegraded":"2024-09-13T14:55:56.567209+0000","last_fullsized":"2024-09-13T14:55:56.567209+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":413,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.339050+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:55:59.339050+0000","last_peered":"2024-09-13T14:55:59.339050+0000","last_clean":"2024-09-13T14:55:59.339050+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:55:59.339050+0000","last_undegraded":"2024-09-13T14:55:59.339050+0000","last_fullsized":"2024-09-13T14:55:59.339050+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342265+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:55:59.342265+0000","last_peered":"2024-09-13T14:55:59.342265+0000","last_clean":"2024-09-13T14:55:59.342265+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:55:59.342265+0000","last_undegraded":"2024-09-13T14:55:59.342265+0000","last_fullsized":"2024-09-13T14:55:59.342265+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":106,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341937+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:55:59.341937+0000","last_peered":"2024-09-13T14:55:59.341937+0000","last_clean":"2024-09-13T14:55:59.341937+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:55:59.341937+0000","last_undegraded":"2024-09-13T14:55:59.341937+0000","last_fullsized":"2024-09-13T14:55:59.341937+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.f","version":"295'14901","reported_seq":15018,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:56:04.425078+0000","last_change":"2024-09-13T14:51:17.035314+0000","last_active":"2024-09-13T14:56:04.425078+0000","last_peered":"2024-09-13T14:56:04.425078+0000","last_clean":"2024-09-13T14:56:04.425078+0000","last_became_active":"2024-09-13T14:51:17.034751+0000","last_became_peered":"2024-09-13T14:51:17.034751+0000","last_unstale":"2024-09-13T14:56:04.425078+0000","last_undegraded":"2024-09-13T14:56:04.425078+0000","last_fullsized":"2024-09-13T14:56:04.425078+0000","mapping_epoch":244,"log_start":"276'8866","ondisk_log_start":"276'8866","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:07.929154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4653056,"num_objects":71,"num_object_clones":0,"num_object_copies":213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14901,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":434,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.334840+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:55:59.334840+0000","last_peered":"2024-09-13T14:55:59.334840+0000","last_clean":"2024-09-13T14:55:59.334840+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:55:59.334840+0000","last_undegraded":"2024-09-13T14:55:59.334840+0000","last_fullsized":"2024-09-13T14:55:59.334840+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":424,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320976+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:55:59.320976+0000","last_peered":"2024-09-13T14:55:59.320976+0000","last_clean":"2024-09-13T14:55:59.320976+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:55:59.320976+0000","last_undegraded":"2024-09-13T14:55:59.320976+0000","last_fullsized":"2024-09-13T14:55:59.320976+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":421,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.362739+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:55:59.362739+0000","last_peered":"2024-09-13T14:55:59.362739+0000","last_clean":"2024-09-13T14:55:59.362739+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:55:59.362739+0000","last_undegraded":"2024-09-13T14:55:59.362739+0000","last_fullsized":"2024-09-13T14:55:59.362739+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":427,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319835+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:55:59.319835+0000","last_peered":"2024-09-13T14:55:59.319835+0000","last_clean":"2024-09-13T14:55:59.319835+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:55:59.319835+0000","last_undegraded":"2024-09-13T14:55:59.319835+0000","last_fullsized":"2024-09-13T14:55:59.319835+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.e","version":"295'14904","reported_seq":17137,"reported_epoch":295,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:56:04.428322+0000","last_change":"2024-09-13T14:54:59.249721+0000","last_active":"2024-09-13T14:56:04.428322+0000","last_peered":"2024-09-13T14:56:04.428322+0000","last_clean":"2024-09-13T14:53:09.526490+0000","last_became_active":"2024-09-13T14:53:11.753975+0000","last_became_peered":"2024-09-13T14:53:11.753975+0000","last_unstale":"2024-09-13T14:56:04.428322+0000","last_undegraded":"2024-09-13T14:53:11.475108+0000","last_fullsized":"2024-09-13T14:53:11.413061+0000","mapping_epoch":271,"log_start":"276'8850","ondisk_log_start":"276'8850","created":244,"last_epoch_clean":245,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:51:15.421529+0000","last_clean_scrub_stamp":"2024-09-13T14:51:15.421529+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4456448,"num_objects":68,"num_object_clones":0,"num_object_copies":204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5815,"num_objects_misplaced":68,"num_objects_unfound":0,"num_objects_dirty":68,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14904,"num_write_kb":479104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1072,"num_bytes_recovered":70123520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":68},{"shards":"1,7","objects":-5815}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":674,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319812+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:55:59.319812+0000","last_peered":"2024-09-13T14:55:59.319812+0000","last_clean":"2024-09-13T14:55:59.319812+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:55:59.319812+0000","last_undegraded":"2024-09-13T14:55:59.319812+0000","last_fullsized":"2024-09-13T14:55:59.319812+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":30,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342396+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:55:59.342396+0000","last_peered":"2024-09-13T14:55:59.342396+0000","last_clean":"2024-09-13T14:55:59.342396+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:55:59.342396+0000","last_undegraded":"2024-09-13T14:55:59.342396+0000","last_fullsized":"2024-09-13T14:55:59.342396+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":279,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268350+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:55:48.268350+0000","last_peered":"2024-09-13T14:55:48.268350+0000","last_clean":"2024-09-13T14:55:48.268350+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:55:48.268350+0000","last_undegraded":"2024-09-13T14:55:48.268350+0000","last_fullsized":"2024-09-13T14:55:48.268350+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":39,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269064+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:55:48.269064+0000","last_peered":"2024-09-13T14:55:48.269064+0000","last_clean":"2024-09-13T14:55:48.269064+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:55:48.269064+0000","last_undegraded":"2024-09-13T14:55:48.269064+0000","last_fullsized":"2024-09-13T14:55:48.269064+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":100,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268425+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:55:48.268425+0000","last_peered":"2024-09-13T14:55:48.268425+0000","last_clean":"2024-09-13T14:55:48.268425+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:55:48.268425+0000","last_undegraded":"2024-09-13T14:55:48.268425+0000","last_fullsized":"2024-09-13T14:55:48.268425+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":98,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268194+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:55:48.268194+0000","last_peered":"2024-09-13T14:55:48.268194+0000","last_clean":"2024-09-13T14:55:48.268194+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:55:48.268194+0000","last_undegraded":"2024-09-13T14:55:48.268194+0000","last_fullsized":"2024-09-13T14:55:48.268194+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":334,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.338317+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:55:59.338317+0000","last_peered":"2024-09-13T14:55:59.338317+0000","last_clean":"2024-09-13T14:55:59.338317+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:55:59.338317+0000","last_undegraded":"2024-09-13T14:55:59.338317+0000","last_fullsized":"2024-09-13T14:55:59.338317+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":372,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.337717+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:55:59.337717+0000","last_peered":"2024-09-13T14:55:59.337717+0000","last_clean":"2024-09-13T14:55:59.337717+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:55:59.337717+0000","last_undegraded":"2024-09-13T14:55:59.337717+0000","last_fullsized":"2024-09-13T14:55:59.337717+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":264,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.269631+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:55:48.269631+0000","last_peered":"2024-09-13T14:55:48.269631+0000","last_clean":"2024-09-13T14:55:48.269631+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:55:48.269631+0000","last_undegraded":"2024-09-13T14:55:48.269631+0000","last_fullsized":"2024-09-13T14:55:48.269631+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":400,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319757+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:55:59.319757+0000","last_peered":"2024-09-13T14:55:59.319757+0000","last_clean":"2024-09-13T14:55:59.319757+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:55:59.319757+0000","last_undegraded":"2024-09-13T14:55:59.319757+0000","last_fullsized":"2024-09-13T14:55:59.319757+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":105,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.347819+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:55:59.347819+0000","last_peered":"2024-09-13T14:55:59.347819+0000","last_clean":"2024-09-13T14:55:59.347819+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:55:59.347819+0000","last_undegraded":"2024-09-13T14:55:59.347819+0000","last_fullsized":"2024-09-13T14:55:59.347819+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":413,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319734+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:55:59.319734+0000","last_peered":"2024-09-13T14:55:59.319734+0000","last_clean":"2024-09-13T14:55:59.319734+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:55:59.319734+0000","last_undegraded":"2024-09-13T14:55:59.319734+0000","last_fullsized":"2024-09-13T14:55:59.319734+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":266,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.268277+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:55:48.268277+0000","last_peered":"2024-09-13T14:55:48.268277+0000","last_clean":"2024-09-13T14:55:48.268277+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:55:48.268277+0000","last_undegraded":"2024-09-13T14:55:48.268277+0000","last_fullsized":"2024-09-13T14:55:48.268277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":457,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.319932+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:55:59.319932+0000","last_peered":"2024-09-13T14:55:59.319932+0000","last_clean":"2024-09-13T14:55:59.319932+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:55:59.319932+0000","last_undegraded":"2024-09-13T14:55:59.319932+0000","last_fullsized":"2024-09-13T14:55:59.319932+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":106,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.342703+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:55:59.342703+0000","last_peered":"2024-09-13T14:55:59.342703+0000","last_clean":"2024-09-13T14:55:59.342703+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:55:59.342703+0000","last_undegraded":"2024-09-13T14:55:59.342703+0000","last_fullsized":"2024-09-13T14:55:59.342703+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":419,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.336699+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:55:59.336699+0000","last_peered":"2024-09-13T14:55:59.336699+0000","last_clean":"2024-09-13T14:55:59.336699+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:55:59.336699+0000","last_undegraded":"2024-09-13T14:55:59.336699+0000","last_fullsized":"2024-09-13T14:55:59.336699+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":420,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320298+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:55:59.320298+0000","last_peered":"2024-09-13T14:55:59.320298+0000","last_clean":"2024-09-13T14:55:59.320298+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:55:59.320298+0000","last_undegraded":"2024-09-13T14:55:59.320298+0000","last_fullsized":"2024-09-13T14:55:59.320298+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":78,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.338458+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:55:59.338458+0000","last_peered":"2024-09-13T14:55:59.338458+0000","last_clean":"2024-09-13T14:55:59.338458+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:55:59.338458+0000","last_undegraded":"2024-09-13T14:55:59.338458+0000","last_fullsized":"2024-09-13T14:55:59.338458+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":439,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.336127+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:55:59.336127+0000","last_peered":"2024-09-13T14:55:59.336127+0000","last_clean":"2024-09-13T14:55:59.336127+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:55:59.336127+0000","last_undegraded":"2024-09-13T14:55:59.336127+0000","last_fullsized":"2024-09-13T14:55:59.336127+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":26,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.341685+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:55:59.341685+0000","last_peered":"2024-09-13T14:55:59.341685+0000","last_clean":"2024-09-13T14:55:59.341685+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:55:59.341685+0000","last_undegraded":"2024-09-13T14:55:59.341685+0000","last_fullsized":"2024-09-13T14:55:59.341685+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":164,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320466+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:55:59.320466+0000","last_peered":"2024-09-13T14:55:59.320466+0000","last_clean":"2024-09-13T14:55:59.320466+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:55:59.320466+0000","last_undegraded":"2024-09-13T14:55:59.320466+0000","last_fullsized":"2024-09-13T14:55:59.320466+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":279,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.675596+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:55:51.675596+0000","last_peered":"2024-09-13T14:55:51.675596+0000","last_clean":"2024-09-13T14:55:51.675596+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:55:51.675596+0000","last_undegraded":"2024-09-13T14:55:51.675596+0000","last_fullsized":"2024-09-13T14:55:51.675596+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":302,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320852+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:55:59.320852+0000","last_peered":"2024-09-13T14:55:59.320852+0000","last_clean":"2024-09-13T14:55:59.320852+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:55:59.320852+0000","last_undegraded":"2024-09-13T14:55:59.320852+0000","last_fullsized":"2024-09-13T14:55:59.320852+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":264,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:51.675632+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:55:51.675632+0000","last_peered":"2024-09-13T14:55:51.675632+0000","last_clean":"2024-09-13T14:55:51.675632+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:55:51.675632+0000","last_undegraded":"2024-09-13T14:55:51.675632+0000","last_fullsized":"2024-09-13T14:55:51.675632+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":459,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320379+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:55:59.320379+0000","last_peered":"2024-09-13T14:55:59.320379+0000","last_clean":"2024-09-13T14:55:59.320379+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:55:59.320379+0000","last_undegraded":"2024-09-13T14:55:59.320379+0000","last_fullsized":"2024-09-13T14:55:59.320379+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":435,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.320953+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:55:59.320953+0000","last_peered":"2024-09-13T14:55:59.320953+0000","last_clean":"2024-09-13T14:55:59.320953+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:55:59.320953+0000","last_undegraded":"2024-09-13T14:55:59.320953+0000","last_fullsized":"2024-09-13T14:55:59.320953+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":294,"state":"active+clean","last_fresh":"2024-09-13T14:55:48.267633+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:55:48.267633+0000","last_peered":"2024-09-13T14:55:48.267633+0000","last_clean":"2024-09-13T14:55:48.267633+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:55:48.267633+0000","last_undegraded":"2024-09-13T14:55:48.267633+0000","last_fullsized":"2024-09-13T14:55:48.267633+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":32,"reported_epoch":295,"state":"active+clean","last_fresh":"2024-09-13T14:55:59.347392+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:55:59.347392+0000","last_peered":"2024-09-13T14:55:59.347392+0000","last_clean":"2024-09-13T14:55:59.347392+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:55:59.347392+0000","last_undegraded":"2024-09-13T14:55:59.347392+0000","last_fullsized":"2024-09-13T14:55:59.347392+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":110559256,"num_objects":1688,"num_object_clones":0,"num_object_copies":5064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16752,"num_objects_misplaced":337,"num_objects_unfound":0,"num_objects_dirty":1688,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":236320,"num_write_kb":7616193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5586,"num_bytes_recovered":365887488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1730031616,"data_stored":1730019400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109516,"ondisk_log_size":109516,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234431,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1076536,"kb_used_data":80688,"kb_used_omap":1,"kb_used_meta":995838,"kb_avail":92668616,"statfs":{"total":95995035648,"available":94892662784,"internally_reserved":0,"allocated":82624512,"data_stored":81003109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1019738571},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496069,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999292,"kb_used_data":101876,"kb_used_omap":1,"kb_used_meta":897406,"kb_avail":92745860,"statfs":{"total":95995035648,"available":94971760640,"internally_reserved":0,"allocated":104321024,"data_stored":102695546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":918944199},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757707,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1129584,"kb_used_data":61544,"kb_used_omap":1,"kb_used_meta":1068030,"kb_avail":92615568,"statfs":{"total":95995035648,"available":94838341632,"internally_reserved":0,"allocated":63021056,"data_stored":61403869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1093663174},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019343,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650556,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589672,"kb_used_data":4320,"kb_used_omap":4,"kb_used_meta":585339,"kb_avail":93155480,"statfs":{"total":95995035648,"available":95391211520,"internally_reserved":0,"allocated":4423680,"data_stored":2814425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":599387695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879490,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547008,"kb_used_data":76088,"kb_used_omap":6,"kb_used_meta":470905,"kb_avail":93198144,"statfs":{"total":95995035648,"available":95434899456,"internally_reserved":0,"allocated":77914112,"data_stored":76285677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":482207289},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435507,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":863392,"kb_used_data":63512,"kb_used_omap":5,"kb_used_meta":799866,"kb_avail":92881760,"statfs":{"total":95995035648,"available":95110922240,"internally_reserved":0,"allocated":65036288,"data_stored":63434086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":819063137},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926082,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":448556,"kb_used_data":37988,"kb_used_omap":5,"kb_used_meta":410554,"kb_avail":93296596,"statfs":{"total":95995035648,"available":95535714304,"internally_reserved":0,"allocated":38899712,"data_stored":37286421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":420407466},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":27787264,"data_stored":27787264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":41549824,"data_stored":41549824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":69472256,"data_stored":69468184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1376194560,"data_stored":1376190488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":48496640,"data_stored":48496640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":97914880,"data_stored":97910808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":68616192,"data_stored":68616192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:56:08.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-13T14:56:08.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.266 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:08.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:08.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:08.389 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:08.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:56:08.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:08.514 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:08.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:08.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:08.632 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:08.639 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:08.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:08.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:08.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:08.865 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:08.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:08.887 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:08.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:08.988 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:09.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:09.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:09.132 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:09.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:09.234 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:09.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:09.358 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:09.380 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:09.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:09.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:09.505 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:09.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:09.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:09.732 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:09.755 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:09.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:09.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:09 smithi042 ceph-mon[113926]: pgmap v1047: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 26 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 16812/1317 objects degraded (1276.538%); 139/1317 objects misplaced (10.554%); 2.9 MiB/s, 51 objects/s recovering 2024-09-13T14:56:09.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:09 smithi042 ceph-mon[108570]: pgmap v1047: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 26 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 16812/1317 objects degraded (1276.538%); 139/1317 objects misplaced (10.554%); 2.9 MiB/s, 51 objects/s recovering 2024-09-13T14:56:09.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:09] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.51.0" 2024-09-13T14:56:09.857 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:09.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:09.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:09.924 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:09 smithi159 ceph-mon[102519]: pgmap v1047: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 26 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 16812/1317 objects degraded (1276.538%); 139/1317 objects misplaced (10.554%); 2.9 MiB/s, 51 objects/s recovering 2024-09-13T14:56:09.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:10.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:10.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:10.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:10.230 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:10.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:10.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:10.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:10.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:10.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:10.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:10.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:10.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:10.820 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16812/1317 objects degraded (1276.538%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:10.831 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:56:10.831 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:65105 10.12 10.b99542b2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:26.464247+0000", 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 524.36478215, 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.86881057100000003, 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.832 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 65105 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464247+0000", 2024-09-13T14:56:10.833 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464247+0000", 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464245+0000", 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:56:10.834 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464288+0000", 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3489999999999999e-05 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464301+0000", 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2401e-05 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.835 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464307+0000", 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6290000000000004e-06 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464364+0000", 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6298000000000002e-05 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.464373+0000", 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9250000000000001e-06 2024-09-13T14:56:10.836 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333045+0000", 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.86867287699999995 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333057+0000", 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1826e-05 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.837 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:65147 10.12 10.47575352 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:26.484285+0000", 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 524.34474358199998, 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.84881990600000001, 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 65147 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.838 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484285+0000", 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484285+0000", 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.839 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484284+0000", 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484496+0000", 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00021164299999999999 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484522+0000", 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5760000000000001e-05 2024-09-13T14:56:10.840 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484529+0000", 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7419999999999997e-06 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484584+0000", 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5099000000000002e-05 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.841 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.484593+0000", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2770000000000005e-06 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333094+0000", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.84850158899999994 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333105+0000", 2024-09-13T14:56:10.842 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0857e-05 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:65304 10.12 10.80b08dd2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e199)", 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:47:26.600823+0000", 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 524.22820614199998, 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73230670099999995, 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.843 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 65304 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600823+0000", 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600823+0000", 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.844 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600821+0000", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600861+0000", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9714e-05 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600877+0000", 2024-09-13T14:56:10.845 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6299000000000001e-05 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600885+0000", 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6809999999999998e-06 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600932+0000", 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6758000000000002e-05 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.846 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:26.600940+0000", 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3809999999999999e-06 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333122+0000", 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73218182499999995 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.847 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.848 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:47:27.333129+0000", 2024-09-13T14:56:10.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16812/1317 objects degraded (1276.538%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:10.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16812/1317 objects degraded (1276.538%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:10.849 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5519999999999999e-06 2024-09-13T14:56:10.849 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.849 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.849 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.849 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155697 10.12 10:4b0ee800:::benchmark_data_smithi042_191918_object32104:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.668701+0000", 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.16032826000003, 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0040385220000001, 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155697 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.850 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.668701+0000", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.668701+0000", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.668699+0000", 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.851 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.668706+0000", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6930000000000007e-06 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669384+0000", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00067787700000000002 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669395+0000", 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0397000000000001e-05 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671233+0000", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001838747 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671241+0000", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9640000000000003e-06 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.672662+0000", 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.001420309 2024-09-13T14:56:10.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.672739+0000", 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7443999999999999e-05 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155708 10.12 10:4b3e20a2:::benchmark_data_smithi042_191918_object32115:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.669403+0000", 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.15962627800002, 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0036708169999999, 2024-09-13T14:56:10.854 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155708 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669403+0000", 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669403+0000", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669402+0000", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669406+0000", 2024-09-13T14:56:10.856 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0629999999999999e-06 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669414+0000", 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1270000000000003e-06 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.669419+0000", 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.519e-06 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671262+0000", 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0018420909999999999 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671269+0000", 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6979999999999998e-06 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.673017+0000", 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.001747894 2024-09-13T14:56:10.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.673073+0000", 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6189e-05 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155716 10.12 10:4eee1c52:::benchmark_data_smithi042_191918_object32123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.670358+0000", 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.15867088599998, 2024-09-13T14:56:10.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.009071214, 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155716 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670358+0000", 2024-09-13T14:56:10.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670358+0000", 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670357+0000", 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670361+0000", 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3780000000000003e-06 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670370+0000", 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4649999999999994e-06 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670377+0000", 2024-09-13T14:56:10.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3250000000000002e-06 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671289+0000", 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00091221399999999995 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671296+0000", 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4309999999999999e-06 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.673108+0000", 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0018122780000001 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.673206+0000", 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8219000000000003e-05 2024-09-13T14:56:10.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.673303+0000", 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6637000000000003e-05 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679234+0000", 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0059308010000000003 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679264+0000", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0369999999999999e-05 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679349+0000", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4661999999999997e-05 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679391+0000", 2024-09-13T14:56:10.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2423999999999999e-05 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679429+0000", 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7966999999999997e-05 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155718 10.12 10:4baa74fd:::benchmark_data_smithi042_191918_object32125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.867 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.670388+0000", 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.15864039000002, 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.009237615, 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155718 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.868 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670388+0000", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670388+0000", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670388+0000", 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670391+0000", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.756e-06 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670402+0000", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1436e-05 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.670408+0000", 2024-09-13T14:56:10.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6609999999999997e-06 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671315+0000", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00090752599999999997 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.671322+0000", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7660000000000001e-06 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.675891+0000", 2024-09-13T14:56:10.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0045692020000001 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.676001+0000", 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010943899999999999 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.676061+0000", 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0368000000000002e-05 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679474+0000", 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034130340000000001 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679538+0000", 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3934000000000001e-05 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679550+0000", 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1643e-05 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679591+0000", 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1383e-05 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679626+0000", 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4916000000000002e-05 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155756 10.12 10:48fcdcb6:::benchmark_data_smithi042_191918_object32163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.710141+0000", 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.11888765999998, 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97763529199999999, 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155756 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.875 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710141+0000", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710141+0000", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710140+0000", 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710145+0000", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7079999999999996e-06 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710156+0000", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1182000000000001e-05 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710163+0000", 2024-09-13T14:56:10.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4649999999999999e-06 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710378+0000", 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00021535599999999999 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.710387+0000", 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2960000000000004e-06 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679685+0000", 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96929728599999998 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679800+0000", 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011506900000000001 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.679902+0000", 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010281999999999999 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687632+0000", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0077297269999999996 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687665+0000", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3151000000000002e-05 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687699+0000", 2024-09-13T14:56:10.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3473999999999998e-05 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687741+0000", 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1903999999999997e-05 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687776+0000", 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5756000000000001e-05 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.881 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155829 10.12 10:4dd3c17d:::benchmark_data_smithi042_191918_object32236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.775691+0000", 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.05333767500002, 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91758638400000003, 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155829 2024-09-13T14:56:10.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775691+0000", 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775691+0000", 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775690+0000", 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775696+0000", 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1079999999999998e-06 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775710+0000", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3662999999999999e-05 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775716+0000", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6080000000000001e-06 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775765+0000", 2024-09-13T14:56:10.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8588999999999998e-05 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.775774+0000", 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5200000000000003e-06 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.682702+0000", 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90692821800000001 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.682810+0000", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010778299999999999 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.682869+0000", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9098999999999998e-05 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687671+0000", 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048018210000000004 2024-09-13T14:56:10.887 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687813+0000", 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014191299999999999 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693174+0000", 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053612399999999998 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693231+0000", 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6332999999999997e-05 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693278+0000", 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6845999999999998e-05 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155839 10.12 10:4a3f17d1:::benchmark_data_smithi042_191918_object32246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.784371+0000", 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.04465731499999, 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90908794500000001, 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155839 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.890 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784371+0000", 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784371+0000", 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784370+0000", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784377+0000", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.64e-06 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784389+0000", 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2006e-05 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784396+0000", 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1420000000000004e-06 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784443+0000", 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7101000000000002e-05 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.784450+0000", 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6969999999999996e-06 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.685115+0000", 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90066536600000002 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.685232+0000", 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117568 2024-09-13T14:56:10.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.685306+0000", 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3891000000000001e-05 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693293+0000", 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0079867470000000006 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693339+0000", 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6195000000000001e-05 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693383+0000", 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3464999999999998e-05 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693421+0000", 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8175e-05 2024-09-13T14:56:10.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693459+0000", 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8458999999999999e-05 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155845 10.12 10:4fc28a17:::benchmark_data_smithi042_191918_object32252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.790855+0000", 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 437.03817417800002, 2024-09-13T14:56:10.897 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90279176900000002, 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155845 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790855+0000", 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790855+0000", 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790853+0000", 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790860+0000", 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6030000000000001e-06 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790872+0000", 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.218e-05 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790880+0000", 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4170000000000003e-06 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790939+0000", 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9218000000000001e-05 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.790948+0000", 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2820000000000002e-06 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687839+0000", 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89689086200000001 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.687944+0000", 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000104764 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.688042+0000", 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8288000000000005e-05 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693348+0000", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053055569999999998 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693499+0000", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015144400000000001 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693572+0000", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3275999999999994e-05 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693611+0000", 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9011999999999999e-05 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693646+0000", 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5148000000000003e-05 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:155966 10.12 10:4f742d89:::benchmark_data_smithi042_191918_object32373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.913377+0000", 2024-09-13T14:56:10.904 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 436.91565185899998, 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.78312174499999998, 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 155966 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913377+0000", 2024-09-13T14:56:10.905 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913377+0000", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913376+0000", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913379+0000", 2024-09-13T14:56:10.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4119999999999999e-06 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913438+0000", 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9558000000000001e-05 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913444+0000", 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3410000000000002e-06 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.907 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.908 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:10.909 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913532+0000", 2024-09-13T14:56:10.909 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7848000000000003e-05 2024-09-13T14:56:10.909 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.909 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.913542+0000", 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0678e-05 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.690298+0000", 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.77675591600000005 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.910 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.690419+0000", 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012104699999999999 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.690837+0000", 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041758700000000003 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.696338+0000", 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0055013950000000001 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.911 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.696345+0000", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3860000000000004e-06 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.696374+0000", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8929e-05 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.696462+0000", 2024-09-13T14:56:10.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8119000000000001e-05 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.696499+0000", 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6977000000000002e-05 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:156033 10.12 10:4b86d665:::benchmark_data_smithi042_191918_object32440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:48:53.977471+0000", 2024-09-13T14:56:10.913 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 436.85155806699998, 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72710907499999999, 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 156033 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977471+0000", 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.914 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977471+0000", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977470+0000", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977473+0000", 2024-09-13T14:56:10.915 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6060000000000001e-06 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977479+0000", 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6189999999999999e-06 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977483+0000", 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3340000000000003e-06 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.916 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977535+0000", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2017e-05 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:53.977550+0000", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5379e-05 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693686+0000", 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.71613559299999996 2024-09-13T14:56:10.917 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693879+0000", 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000192642 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.693984+0000", 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000105389 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.918 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.701843+0000", 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0078588449999999997 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.704411+0000", 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002568271 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.920 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.704464+0000", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3443000000000003e-05 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.704544+0000", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9355000000000002e-05 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:48:54.704580+0000", 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5982999999999998e-05 2024-09-13T14:56:10.921 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213300 10.16 10.2fc051b6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 378.04249464700001, 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.004492741, 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.922 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213300 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786533+0000", 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786539+0000", 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4999999999999996e-06 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786551+0000", 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1389000000000001e-05 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786557+0000", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5359999999999998e-06 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789016+0000", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002459265 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789023+0000", 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.443e-06 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791016+0000", 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.001992759 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791027+0000", 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1364e-05 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213303 10.16 10.11a593b6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.926 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 378.04246244400002, 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0044890209999999, 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213303 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:56:10.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786569+0000", 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9100000000000001e-06 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786575+0000", 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786580+0000", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3489999999999999e-06 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789039+0000", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002458876 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789043+0000", 2024-09-13T14:56:10.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3710000000000002e-06 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791048+0000", 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0020046730000001 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791055+0000", 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4549999999999998e-06 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.931 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213349 10.16 10.8c4fc176 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 378.00657867500001, 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96863077099999995, 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213349 2024-09-13T14:56:10.932 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822449+0000", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822454+0000", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5799999999999999e-06 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822467+0000", 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2092e-05 2024-09-13T14:56:10.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822472+0000", 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8259999999999998e-06 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822516+0000", 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3659000000000003e-05 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822523+0000", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3300000000000001e-06 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791073+0000", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96854951199999995 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791081+0000", 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.011e-06 2024-09-13T14:56:10.936 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213416 10.16 10.6199b816 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 377.93252205599998, 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89460015299999995, 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.937 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213416 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896505+0000", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896517+0000", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2329e-05 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896530+0000", 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2782999999999999e-05 2024-09-13T14:56:10.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896537+0000", 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1969999999999999e-06 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896745+0000", 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020745599999999999 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896752+0000", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0679999999999999e-06 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791100+0000", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89434780599999997 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791107+0000", 2024-09-13T14:56:10.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0949999999999998e-06 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213480 10.16 10.2728f336 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 377.87268503600001, 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.83479029299999996, 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.942 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213480 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956343+0000", 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956347+0000", 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2989999999999998e-06 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956356+0000", 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6829999999999997e-06 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956362+0000", 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2210000000000002e-06 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956625+0000", 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00026299400000000001 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956635+0000", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0190999999999999e-05 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791127+0000", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.83449121999999998 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791134+0000", 2024-09-13T14:56:10.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4240000000000004e-06 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213511 10.16 10.8fe7b616 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 377.83900403600001, 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80113405299999996, 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.947 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213511 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990023+0000", 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990030+0000", 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0700000000000001e-06 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990043+0000", 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2337000000000001e-05 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990049+0000", 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6769999999999998e-06 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990108+0000", 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8445e-05 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990116+0000", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2160000000000006e-06 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791151+0000", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80103517700000004 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791159+0000", 2024-09-13T14:56:10.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.515e-06 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213544 10.16 10.adf34f56 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 377.80283457600001, 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76498887800000004, 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:56:10.952 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213544 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:56:10.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026193+0000", 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026197+0000", 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9570000000000004e-06 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:56:10.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026206+0000", 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4130000000000006e-06 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026212+0000", 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0920000000000003e-06 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026252+0000", 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0210000000000003e-05 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026259+0000", 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2080000000000003e-06 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791176+0000", 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76491660800000005 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:56:10.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791183+0000", 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1219999999999997e-06 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:56:10.957 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:56:10.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:10.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:10.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.080 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.102 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:11.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:11.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:11.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:11.339 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-09-13T14:56:11.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-09-13T14:56:11.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:11.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:11.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:11.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:11.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:11.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:11.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:11.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:11.722 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:11.824 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:11.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:11.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:11.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:11 smithi042 ceph-mon[113926]: pgmap v1048: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 16847/78 objects degraded (21598.718%); 2.8 MiB/s, 71 objects/s recovering 2024-09-13T14:56:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:11 smithi042 ceph-mon[108570]: pgmap v1048: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 16847/78 objects degraded (21598.718%); 2.8 MiB/s, 71 objects/s recovering 2024-09-13T14:56:11.947 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:11.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:11.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:11 smithi159 ceph-mon[102519]: pgmap v1048: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 577 op/s; 16847/78 objects degraded (21598.718%); 2.8 MiB/s, 71 objects/s recovering 2024-09-13T14:56:12.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:12.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:12.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.220 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:12.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:12.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:12.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:12.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:12.698 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:12.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:12.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:12.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:12.845 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:12.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:12.947 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:12.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:12.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:12.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:13.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:13.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:13.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:13.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:13.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:13.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:13.342 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:13.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:13.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:13.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:13.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.591 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:13.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.693 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:13.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:13.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:13.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:13.841 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:13.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:13 smithi042 ceph-mon[113926]: pgmap v1049: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 16810/78 objects degraded (21551.282%); 2.1 MiB/s, 98 objects/s recovering 2024-09-13T14:56:13.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:13 smithi042 ceph-mon[108570]: pgmap v1049: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 16810/78 objects degraded (21551.282%); 2.1 MiB/s, 98 objects/s recovering 2024-09-13T14:56:13.943 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:13.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:13.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:13 smithi159 ceph-mon[102519]: pgmap v1049: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 16810/78 objects degraded (21551.282%); 2.1 MiB/s, 98 objects/s recovering 2024-09-13T14:56:14.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:14.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:14.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:14.213 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:14.315 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:14.337 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:14.439 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:14.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:14.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:14.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:14.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:14.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:14.935 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:14.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:14.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:15.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:15.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:15.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:15.105 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:15.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:15.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:15.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:15.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.399 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:15.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:15.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:15.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:15.556 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:15.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:15.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:56:15.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:15.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:15.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:15.807 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:15 smithi042 ceph-mon[113926]: pgmap v1050: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 16668/78 objects degraded (21369.231%); 1.3 MiB/s, 120 objects/s recovering 2024-09-13T14:56:15.807 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16668/78 objects degraded (21369.231%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:15.808 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:15 smithi042 ceph-mon[108570]: pgmap v1050: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 16668/78 objects degraded (21369.231%); 1.3 MiB/s, 120 objects/s recovering 2024-09-13T14:56:15.808 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16668/78 objects degraded (21369.231%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:15.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:15.834 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:15.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:15.956 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:15 smithi159 ceph-mon[102519]: pgmap v1050: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 288 op/s; 16668/78 objects degraded (21369.231%); 1.3 MiB/s, 120 objects/s recovering 2024-09-13T14:56:15.956 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16668/78 objects degraded (21369.231%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:15.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:15.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:16.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:16.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:16.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:16.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:16.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:16.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:16.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:16.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:16.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:16.460 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:16.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:16.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:16.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:16.708 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:16.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:16.834 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:16.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:16.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:16.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:16.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:16.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:17.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:17.086 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:17.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:17.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:17.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:17.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:17.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:17.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:17.459 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:17.560 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:17.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:17.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:17.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:17.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:17.826 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:17 smithi159 ceph-mon[102519]: pgmap v1051: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 16668/78 objects degraded (21369.231%); 912 KiB/s, 113 objects/s recovering 2024-09-13T14:56:17.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:17.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:17.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:17 smithi042 ceph-mon[113926]: pgmap v1051: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 16668/78 objects degraded (21369.231%); 912 KiB/s, 113 objects/s recovering 2024-09-13T14:56:17.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:17 smithi042 ceph-mon[108570]: pgmap v1051: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 205 op/s; 16668/78 objects degraded (21369.231%); 912 KiB/s, 113 objects/s recovering 2024-09-13T14:56:17.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:17.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:17.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:17.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:18.006 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:18.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:18.091 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:18.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:18.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:18.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:18.230 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:56:18.231 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:56:18.231 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T14:56:18.231 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:56:18.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:18.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:18.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:18.465 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:18.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:18.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:18.578 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:56:18.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:18.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:18.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:18.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:18.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:18 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:18.822 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:18 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:18.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:18.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:18.925 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:18 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:56:18.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:56:18.950 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:18.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:18.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:19.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.164 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:19.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:19.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:19.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:19.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:19.352 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:19.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.601 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:19.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:19.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:19.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[108570]: pgmap v1052: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 16153/78 objects degraded (20708.974%); 912 KiB/s, 156 objects/s recovering 2024-09-13T14:56:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:19.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[108570]: osdmap e296: 8 total, 8 up, 7 in 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1770841798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[113926]: pgmap v1052: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 16153/78 objects degraded (20708.974%); 912 KiB/s, 156 objects/s recovering 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[113926]: osdmap e296: 8 total, 8 up, 7 in 2024-09-13T14:56:19.730 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1770841798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:19.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:19.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:19.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:19.977 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:19.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:19.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:19.991 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:19] "GET /metrics HTTP/1.1" 200 36479 "" "Prometheus/2.51.0" 2024-09-13T14:56:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:19 smithi159 ceph-mon[102519]: pgmap v1052: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 140 op/s; 16153/78 objects degraded (20708.974%); 912 KiB/s, 156 objects/s recovering 2024-09-13T14:56:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:19 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:56:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1145676577' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:56:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:19 smithi159 ceph-mon[102519]: osdmap e296: 8 total, 8 up, 7 in 2024-09-13T14:56:19.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1770841798' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:20.078 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:20.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:20.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:20.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:20.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:20.285 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:56:20.285 INFO:teuthology.orchestra.run.smithi042.stdout:{"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16153/78 objects degraded (20708.974%), 3 pgs degraded, 3 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":32,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1544,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":296,"num_osds":8,"num_up_osds":8,"osd_up_since":1726238436,"num_in_osds":7,"osd_in_since":1726239189,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":94},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":26,"data_bytes":595941,"bytes_used":5402148864,"bytes_avail":666563100672,"bytes_total":671965249536,"degraded_objects":16153,"degraded_total":78,"degraded_ratio":207.08974358974359,"recovering_objects_per_sec":156,"recovering_bytes_per_sec":933651,"recovering_keys_per_sec":0,"num_objects_recovered":1878,"num_bytes_recovered":11206656,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":140},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi042.pkkbnw","status":"up:active","gid":14652}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"}},"servicemap":{"epoch":26,"modified":"2024-09-13T14:50:44.357827+0000","services":{}},"progress_events":{}} 2024-09-13T14:56:20.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:20.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:20.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:20.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:20.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:56:20.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:56:20.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:56:20.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:56:20.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:56:20.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:56:20.622 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:20.622 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:20.622 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:20.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:20.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:20.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:20.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1939080020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:56:20.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 16153/78 objects degraded (20708.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:20.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1939080020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:56:20.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 16153/78 objects degraded (20708.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:20.857 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:20.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:20.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:20.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:20.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1939080020' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-13T14:56:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 16153/78 objects degraded (20708.974%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:21.083 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:21.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:21.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.231 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:21.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:21.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:21.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:21.826 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:21 smithi042 ceph-mon[113926]: pgmap v1054: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15730/78 objects degraded (20166.667%); 0 B/s, 182 objects/s recovering 2024-09-13T14:56:21.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:21.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:21 smithi042 ceph-mon[108570]: pgmap v1054: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15730/78 objects degraded (20166.667%); 0 B/s, 182 objects/s recovering 2024-09-13T14:56:21.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:21.850 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:21.952 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:21.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:21.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:21 smithi159 ceph-mon[102519]: pgmap v1054: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15730/78 objects degraded (20166.667%); 0 B/s, 182 objects/s recovering 2024-09-13T14:56:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:22.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:22.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:22.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:22.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:22.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:22.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:22.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:22.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:22.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:22.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:22.593 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:22.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:22.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:22.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:22.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:22.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:22.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:22.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:22.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.087 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:23.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:23.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:23.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:23.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:23.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:23.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:23.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:23.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.456 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:23.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:23.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:23.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:23.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:23.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:23.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:23.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:23.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:23.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:23 smithi042 ceph-mon[108570]: pgmap v1055: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15089/78 objects degraded (19344.872%); 0 B/s, 199 objects/s recovering 2024-09-13T14:56:23.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:23 smithi042 ceph-mon[113926]: pgmap v1055: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15089/78 objects degraded (19344.872%); 0 B/s, 199 objects/s recovering 2024-09-13T14:56:23.927 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:23.950 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:23.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:23 smithi159 ceph-mon[102519]: pgmap v1055: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 15089/78 objects degraded (19344.872%); 0 B/s, 199 objects/s recovering 2024-09-13T14:56:24.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.075 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:24.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:24.199 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:24.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:24.323 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:24.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:24.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:24.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:24.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:24.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:24.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:24.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:24.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:24.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:24.821 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:24.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:24.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:24.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:25.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.071 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:25.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:25.173 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:25.322 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:25.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:25.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:25.548 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:25.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:25.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:25.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:25.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:25.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:25.818 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:25 smithi042 ceph-mon[108570]: pgmap v1056: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:25.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14630/78 objects degraded (18756.410%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:25 smithi042 ceph-mon[113926]: pgmap v1056: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:25.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14630/78 objects degraded (18756.410%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:25.887 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:25.887 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:25.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:25 smithi159 ceph-mon[102519]: pgmap v1056: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:25.887 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14630/78 objects degraded (18756.410%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:25.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:25.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:25.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:26.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:26.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:26.189 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:26.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:26.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:26.312 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:26.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:26.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:26.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:26.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:26.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:26.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:26.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:56:26.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:26.933 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:26.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:26.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:27.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:27.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:27.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:27.092 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:56:27.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:27.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:27.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:27.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:27.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:27.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:27.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:27.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:27.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:27.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:27.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:27.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:27.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:27.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:27.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:27 smithi042 ceph-mon[108570]: pgmap v1057: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:27.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:27 smithi042 ceph-mon[113926]: pgmap v1057: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:27.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:27.969 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:27.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:27.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:27 smithi159 ceph-mon[102519]: pgmap v1057: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 14630/78 objects degraded (18756.410%); 0 B/s, 203 objects/s recovering 2024-09-13T14:56:28.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:28.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:28.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:28.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:28.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:28.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:28.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:28.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:28.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:28.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:28.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:28.945 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:28.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:28.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:28.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:28.975 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:28.976 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:29.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:29.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:29.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:29.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:29.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:29.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:29.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:29.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:29.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:29.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:29.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:29.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T14:56:29.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:29.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:29.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:29 smithi042 ceph-mon[113926]: pgmap v1058: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 13975/78 objects degraded (17916.667%); 0 B/s, 217 objects/s recovering 2024-09-13T14:56:29.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:29 smithi042 ceph-mon[108570]: pgmap v1058: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 13975/78 objects degraded (17916.667%); 0 B/s, 217 objects/s recovering 2024-09-13T14:56:29.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:29.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:29.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:29.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:29.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:29.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:29.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:29 smithi159 ceph-mon[102519]: pgmap v1058: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 94 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 13975/78 objects degraded (17916.667%); 0 B/s, 217 objects/s recovering 2024-09-13T14:56:30.056 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:29] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.51.0" 2024-09-13T14:56:30.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:30.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:30.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.219 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:30.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:30.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:30.385 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-13T14:56:30.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:56:30.399 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:30.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:30.457 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_6' removed 2024-09-13T14:56:30.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:30.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:30.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:30.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:30.686 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/466110058' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.686 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.686 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[113926]: osdmap e297: 8 total, 8 up, 7 in 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/466110058' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:56:30.687 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:30 smithi042 ceph-mon[108570]: osdmap e297: 8 total, 8 up, 7 in 2024-09-13T14:56:30.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:30.800 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:30.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:30.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:30.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:30.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:30 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/466110058' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T14:56:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-13T14:56:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T14:56:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:30 smithi159 ceph-mon[102519]: osdmap e297: 8 total, 8 up, 7 in 2024-09-13T14:56:31.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.050 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:31.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:31.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:31.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:31.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:31.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:31.401 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:31.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.423 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:56:31.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:31.557 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:31.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:31.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:31.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:31.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:31 smithi042 ceph-mon[108570]: pgmap v1060: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:31.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:31 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized) 2024-09-13T14:56:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:31 smithi042 ceph-mon[113926]: pgmap v1060: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:31 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized) 2024-09-13T14:56:31.906 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:31.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:31.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:31.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:31.989 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T14:56:31.990 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T14:56:31.990 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_7 2024-09-13T14:56:31.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_7 16 2024-09-13T14:56:31.998 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:31 smithi159 ceph-mon[102519]: pgmap v1060: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:31 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13975/78 objects degraded (17916.667%), 3 pgs degraded, 3 pgs undersized) 2024-09-13T14:56:32.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.050 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:32.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:32.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:32.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:32.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:32.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:32.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:32.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:32.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:32.775 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:32.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:32.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:32.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:32.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:32.924 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:32.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:33.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.048 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:33.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:33.172 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:33.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:33.274 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.297 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:33.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:33.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:33.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.778 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:33.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:33.805 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:33.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:33.812 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:33.812 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:33.818 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:56:33.819 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:33.839 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:56:33.840 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:33.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:33 smithi042 ceph-mon[108570]: pgmap v1061: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:33.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:33 smithi042 ceph-mon[113926]: pgmap v1061: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:33.861 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:56:33.862 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:33.878 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:56:33.879 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:33.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:33.928 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:33.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:33 smithi159 ceph-mon[102519]: pgmap v1061: 83 pgs: 83 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:34.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.056 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:34.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:34.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:34.282 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:34.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:34.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:34.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:34.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:34.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:34.558 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:34.660 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:34.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:34.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:34.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:34.813 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:34.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:34.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:34.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:34.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1245453735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:35.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:35.071 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:35.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1245453735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1245453735' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:35.173 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:35.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:35.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:35.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:35.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:35.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:35.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:35.557 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:35.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:35.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:35.644 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:35.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:35.708 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:35.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:35.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:35.834 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:35.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:35.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:35.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:35.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:35.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:35 smithi159 ceph-mon[102519]: pgmap v1062: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:36.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.085 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:36.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:35 smithi042 ceph-mon[113926]: pgmap v1062: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:35 smithi042 ceph-mon[108570]: pgmap v1062: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:36.186 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:36.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:36.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:36.341 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:36.442 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:36.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:36.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:36.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:36.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:36.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:36.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:36.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:36.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:36.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:36.960 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:36.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:37.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:37.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:37.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:37.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:37.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:37.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:37.460 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:37.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:37.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:37.709 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:37.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:37.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:37.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:37.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:37.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:37.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:37.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:37.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.061 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:37 smithi042 ceph-mon[113926]: pgmap v1063: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:38.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:37 smithi042 ceph-mon[108570]: pgmap v1063: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:38.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:56:38.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:38.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:37 smithi159 ceph-mon[102519]: pgmap v1063: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:38.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:38.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:38.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:38.457 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:38.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:38.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:38.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:38.708 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:38.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:38.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:38.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:38.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:38.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:38.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:39.084 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:39.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:39.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:39.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:39.456 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:39.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:39.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:39.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:39.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:39.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:39.740 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:56:39.741 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:56:39.741 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T14:56:39.741 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:56:39.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:39.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:39.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:39.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:39.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:40.016 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:39 smithi159 ceph-mon[102519]: pgmap v1064: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:40.017 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:40.052 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:39 smithi042 ceph-mon[108570]: pgmap v1064: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:40.052 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:40.052 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:39 smithi042 ceph-mon[113926]: pgmap v1064: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:40.052 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:56:40.053 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:39] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.51.0" 2024-09-13T14:56:40.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:40.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:40.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:40.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:40.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:40.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.330 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:40.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:40.341 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:56:40.342 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:56:40.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:40.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:40.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:40.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:40.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.699 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:40.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:40.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:40.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:40.946 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:40.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:41.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:41.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:41.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:41.206 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:41.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:41.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:41.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:41.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:41.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:41.568 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:41.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:41.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:41.686 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1064,"stamp":"2024-09-13T14:56:38.499288+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5289672,"kb_used_data":28212,"kb_used_omap":27,"kb_used_meta":5261348,"kb_avail":650926392,"statfs":{"total":671965249536,"available":666548625408,"internally_reserved":0,"allocated":28889088,"data_stored":17472993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5387621323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":33,"apply_latency_ms":33,"commit_latency_ns":33000000,"apply_latency_ns":33000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-13975,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-238008,"num_write_kb":-7616193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-9642,"num_bytes_recovered":-377094144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004216"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":417,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.572581+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:56:30.572581+0000","last_peered":"2024-09-13T14:56:30.572581+0000","last_clean":"2024-09-13T14:56:30.572581+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:56:30.572581+0000","last_undegraded":"2024-09-13T14:56:30.572581+0000","last_fullsized":"2024-09-13T14:56:30.572581+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":84,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453128+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:56:30.453128+0000","last_peered":"2024-09-13T14:56:30.453128+0000","last_clean":"2024-09-13T14:56:30.453128+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:56:30.453128+0000","last_undegraded":"2024-09-13T14:56:30.453128+0000","last_fullsized":"2024-09-13T14:56:30.453128+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":265,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.590159+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:56:18.590159+0000","last_peered":"2024-09-13T14:56:18.590159+0000","last_clean":"2024-09-13T14:56:18.590159+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:56:18.590159+0000","last_undegraded":"2024-09-13T14:56:18.590159+0000","last_fullsized":"2024-09-13T14:56:18.590159+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":110,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459176+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:56:30.459176+0000","last_peered":"2024-09-13T14:56:30.459176+0000","last_clean":"2024-09-13T14:56:30.459176+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:56:30.459176+0000","last_undegraded":"2024-09-13T14:56:30.459176+0000","last_fullsized":"2024-09-13T14:56:30.459176+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":110,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.463332+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:56:30.463332+0000","last_peered":"2024-09-13T14:56:30.463332+0000","last_clean":"2024-09-13T14:56:30.463332+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:56:30.463332+0000","last_undegraded":"2024-09-13T14:56:30.463332+0000","last_fullsized":"2024-09-13T14:56:30.463332+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":41,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512376+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:56:19.512376+0000","last_peered":"2024-09-13T14:56:19.512376+0000","last_clean":"2024-09-13T14:56:19.512376+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:56:19.512376+0000","last_undegraded":"2024-09-13T14:56:19.512376+0000","last_fullsized":"2024-09-13T14:56:19.512376+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":443,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.573261+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:56:30.573261+0000","last_peered":"2024-09-13T14:56:30.573261+0000","last_clean":"2024-09-13T14:56:30.573261+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:56:30.573261+0000","last_undegraded":"2024-09-13T14:56:30.573261+0000","last_fullsized":"2024-09-13T14:56:30.573261+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":352,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512003+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:56:19.512003+0000","last_peered":"2024-09-13T14:56:19.512003+0000","last_clean":"2024-09-13T14:56:19.512003+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:56:19.512003+0000","last_undegraded":"2024-09-13T14:56:19.512003+0000","last_fullsized":"2024-09-13T14:56:19.512003+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":34,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.458666+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:56:30.458666+0000","last_peered":"2024-09-13T14:56:30.458666+0000","last_clean":"2024-09-13T14:56:30.458666+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:56:30.458666+0000","last_undegraded":"2024-09-13T14:56:30.458666+0000","last_fullsized":"2024-09-13T14:56:30.458666+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589688+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:56:18.589688+0000","last_peered":"2024-09-13T14:56:18.589688+0000","last_clean":"2024-09-13T14:56:18.589688+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:56:18.589688+0000","last_undegraded":"2024-09-13T14:56:18.589688+0000","last_fullsized":"2024-09-13T14:56:18.589688+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":93,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.477390+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:56:30.477390+0000","last_peered":"2024-09-13T14:56:30.477390+0000","last_clean":"2024-09-13T14:56:30.477390+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:56:30.477390+0000","last_undegraded":"2024-09-13T14:56:30.477390+0000","last_fullsized":"2024-09-13T14:56:30.477390+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":314,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453414+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:56:30.453414+0000","last_peered":"2024-09-13T14:56:30.453414+0000","last_clean":"2024-09-13T14:56:30.453414+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:56:30.453414+0000","last_undegraded":"2024-09-13T14:56:30.453414+0000","last_fullsized":"2024-09-13T14:56:30.453414+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":169,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.590028+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:56:18.590028+0000","last_peered":"2024-09-13T14:56:18.590028+0000","last_clean":"2024-09-13T14:56:18.590028+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:56:18.590028+0000","last_undegraded":"2024-09-13T14:56:18.590028+0000","last_fullsized":"2024-09-13T14:56:18.590028+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":268,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512238+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:56:19.512238+0000","last_peered":"2024-09-13T14:56:19.512238+0000","last_clean":"2024-09-13T14:56:19.512238+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:56:19.512238+0000","last_undegraded":"2024-09-13T14:56:19.512238+0000","last_fullsized":"2024-09-13T14:56:19.512238+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":267,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589958+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:56:18.589958+0000","last_peered":"2024-09-13T14:56:18.589958+0000","last_clean":"2024-09-13T14:56:18.589958+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:56:18.589958+0000","last_undegraded":"2024-09-13T14:56:18.589958+0000","last_fullsized":"2024-09-13T14:56:18.589958+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":117,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.467265+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:56:30.467265+0000","last_peered":"2024-09-13T14:56:30.467265+0000","last_clean":"2024-09-13T14:56:30.467265+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:56:30.467265+0000","last_undegraded":"2024-09-13T14:56:30.467265+0000","last_fullsized":"2024-09-13T14:56:30.467265+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":768,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.576727+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:56:30.576727+0000","last_peered":"2024-09-13T14:56:30.576727+0000","last_clean":"2024-09-13T14:56:30.576727+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:56:30.576727+0000","last_undegraded":"2024-09-13T14:56:30.576727+0000","last_fullsized":"2024-09-13T14:56:30.576727+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":461,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.582448+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:56:30.582448+0000","last_peered":"2024-09-13T14:56:30.582448+0000","last_clean":"2024-09-13T14:56:30.582448+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:56:30.582448+0000","last_undegraded":"2024-09-13T14:56:30.582448+0000","last_fullsized":"2024-09-13T14:56:30.582448+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.572749+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:56:30.572749+0000","last_peered":"2024-09-13T14:56:30.572749+0000","last_clean":"2024-09-13T14:56:30.572749+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:56:30.572749+0000","last_undegraded":"2024-09-13T14:56:30.572749+0000","last_fullsized":"2024-09-13T14:56:30.572749+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":537,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589530+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:56:18.589530+0000","last_peered":"2024-09-13T14:56:18.589530+0000","last_clean":"2024-09-13T14:56:18.589530+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:56:18.589530+0000","last_undegraded":"2024-09-13T14:56:18.589530+0000","last_fullsized":"2024-09-13T14:56:18.589530+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":307,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512364+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:56:19.512364+0000","last_peered":"2024-09-13T14:56:19.512364+0000","last_clean":"2024-09-13T14:56:19.512364+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:56:19.512364+0000","last_undegraded":"2024-09-13T14:56:19.512364+0000","last_fullsized":"2024-09-13T14:56:19.512364+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.455801+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:56:30.455801+0000","last_peered":"2024-09-13T14:56:30.455801+0000","last_clean":"2024-09-13T14:56:30.455801+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:56:30.455801+0000","last_undegraded":"2024-09-13T14:56:30.455801+0000","last_fullsized":"2024-09-13T14:56:30.455801+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":148,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453649+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:56:30.453649+0000","last_peered":"2024-09-13T14:56:30.453649+0000","last_clean":"2024-09-13T14:56:30.453649+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:56:30.453649+0000","last_undegraded":"2024-09-13T14:56:30.453649+0000","last_fullsized":"2024-09-13T14:56:30.453649+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":129,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512506+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:56:19.512506+0000","last_peered":"2024-09-13T14:56:19.512506+0000","last_clean":"2024-09-13T14:56:19.512506+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:56:19.512506+0000","last_undegraded":"2024-09-13T14:56:19.512506+0000","last_fullsized":"2024-09-13T14:56:19.512506+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":537,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589666+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:56:18.589666+0000","last_peered":"2024-09-13T14:56:18.589666+0000","last_clean":"2024-09-13T14:56:18.589666+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:56:18.589666+0000","last_undegraded":"2024-09-13T14:56:18.589666+0000","last_fullsized":"2024-09-13T14:56:18.589666+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":420,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.597085+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:56:30.597085+0000","last_peered":"2024-09-13T14:56:30.597085+0000","last_clean":"2024-09-13T14:56:30.597085+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:56:30.597085+0000","last_undegraded":"2024-09-13T14:56:30.597085+0000","last_fullsized":"2024-09-13T14:56:30.597085+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":426,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.485427+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:56:30.485427+0000","last_peered":"2024-09-13T14:56:30.485427+0000","last_clean":"2024-09-13T14:56:30.485427+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:56:30.485427+0000","last_undegraded":"2024-09-13T14:56:30.485427+0000","last_fullsized":"2024-09-13T14:56:30.485427+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":494,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.458362+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:56:30.458362+0000","last_peered":"2024-09-13T14:56:30.458362+0000","last_clean":"2024-09-13T14:56:30.458362+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:56:30.458362+0000","last_undegraded":"2024-09-13T14:56:30.458362+0000","last_fullsized":"2024-09-13T14:56:30.458362+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":282,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589883+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:56:18.589883+0000","last_peered":"2024-09-13T14:56:18.589883+0000","last_clean":"2024-09-13T14:56:18.589883+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:56:18.589883+0000","last_undegraded":"2024-09-13T14:56:18.589883+0000","last_fullsized":"2024-09-13T14:56:18.589883+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":405,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.580876+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:56:30.580876+0000","last_peered":"2024-09-13T14:56:30.580876+0000","last_clean":"2024-09-13T14:56:30.580876+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:56:30.580876+0000","last_undegraded":"2024-09-13T14:56:30.580876+0000","last_fullsized":"2024-09-13T14:56:30.580876+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:21.571671+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:56:21.571671+0000","last_peered":"2024-09-13T14:56:21.571671+0000","last_clean":"2024-09-13T14:56:21.571671+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:56:21.571671+0000","last_undegraded":"2024-09-13T14:56:21.571671+0000","last_fullsized":"2024-09-13T14:56:21.571671+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":689,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.470811+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:56:30.470811+0000","last_peered":"2024-09-13T14:56:30.470811+0000","last_clean":"2024-09-13T14:56:30.470811+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:56:30.470811+0000","last_undegraded":"2024-09-13T14:56:30.470811+0000","last_fullsized":"2024-09-13T14:56:30.470811+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":406,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.470918+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:56:30.470918+0000","last_peered":"2024-09-13T14:56:30.470918+0000","last_clean":"2024-09-13T14:56:30.470918+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:56:30.470918+0000","last_undegraded":"2024-09-13T14:56:30.470918+0000","last_fullsized":"2024-09-13T14:56:30.470918+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.476037+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:56:30.476037+0000","last_peered":"2024-09-13T14:56:30.476037+0000","last_clean":"2024-09-13T14:56:30.476037+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:56:30.476037+0000","last_undegraded":"2024-09-13T14:56:30.476037+0000","last_fullsized":"2024-09-13T14:56:30.476037+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":30,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459140+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:56:30.459140+0000","last_peered":"2024-09-13T14:56:30.459140+0000","last_clean":"2024-09-13T14:56:30.459140+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:56:30.459140+0000","last_undegraded":"2024-09-13T14:56:30.459140+0000","last_fullsized":"2024-09-13T14:56:30.459140+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":607,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459493+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:56:30.459493+0000","last_peered":"2024-09-13T14:56:30.459493+0000","last_clean":"2024-09-13T14:56:30.459493+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:56:30.459493+0000","last_undegraded":"2024-09-13T14:56:30.459493+0000","last_fullsized":"2024-09-13T14:56:30.459493+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":457,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453819+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:56:30.453819+0000","last_peered":"2024-09-13T14:56:30.453819+0000","last_clean":"2024-09-13T14:56:30.453819+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:56:30.453819+0000","last_undegraded":"2024-09-13T14:56:30.453819+0000","last_fullsized":"2024-09-13T14:56:30.453819+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":418,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.471895+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:56:30.471895+0000","last_peered":"2024-09-13T14:56:30.471895+0000","last_clean":"2024-09-13T14:56:30.471895+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:56:30.471895+0000","last_undegraded":"2024-09-13T14:56:30.471895+0000","last_fullsized":"2024-09-13T14:56:30.471895+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":468,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.485267+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:56:30.485267+0000","last_peered":"2024-09-13T14:56:30.485267+0000","last_clean":"2024-09-13T14:56:30.485267+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:56:30.485267+0000","last_undegraded":"2024-09-13T14:56:30.485267+0000","last_fullsized":"2024-09-13T14:56:30.485267+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":571,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589710+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:56:18.589710+0000","last_peered":"2024-09-13T14:56:18.589710+0000","last_clean":"2024-09-13T14:56:18.589710+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:56:18.589710+0000","last_undegraded":"2024-09-13T14:56:18.589710+0000","last_fullsized":"2024-09-13T14:56:18.589710+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":283,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589873+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:56:18.589873+0000","last_peered":"2024-09-13T14:56:18.589873+0000","last_clean":"2024-09-13T14:56:18.589873+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:56:18.589873+0000","last_undegraded":"2024-09-13T14:56:18.589873+0000","last_fullsized":"2024-09-13T14:56:18.589873+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":314,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.466283+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:56:30.466283+0000","last_peered":"2024-09-13T14:56:30.466283+0000","last_clean":"2024-09-13T14:56:30.466283+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:56:30.466283+0000","last_undegraded":"2024-09-13T14:56:30.466283+0000","last_fullsized":"2024-09-13T14:56:30.466283+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":430,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.472742+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:56:30.472742+0000","last_peered":"2024-09-13T14:56:30.472742+0000","last_clean":"2024-09-13T14:56:30.472742+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:56:30.472742+0000","last_undegraded":"2024-09-13T14:56:30.472742+0000","last_fullsized":"2024-09-13T14:56:30.472742+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":545,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512525+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:56:19.512525+0000","last_peered":"2024-09-13T14:56:19.512525+0000","last_clean":"2024-09-13T14:56:19.512525+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:56:19.512525+0000","last_undegraded":"2024-09-13T14:56:19.512525+0000","last_fullsized":"2024-09-13T14:56:19.512525+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":284,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:21.706381+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:56:21.706381+0000","last_peered":"2024-09-13T14:56:21.706381+0000","last_clean":"2024-09-13T14:56:21.706381+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:56:21.706381+0000","last_undegraded":"2024-09-13T14:56:21.706381+0000","last_fullsized":"2024-09-13T14:56:21.706381+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":482,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.474055+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:56:30.474055+0000","last_peered":"2024-09-13T14:56:30.474055+0000","last_clean":"2024-09-13T14:56:30.474055+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:56:30.474055+0000","last_undegraded":"2024-09-13T14:56:30.474055+0000","last_fullsized":"2024-09-13T14:56:30.474055+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":346,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.472116+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:56:30.472116+0000","last_peered":"2024-09-13T14:56:30.472116+0000","last_clean":"2024-09-13T14:56:30.472116+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:56:30.472116+0000","last_undegraded":"2024-09-13T14:56:30.472116+0000","last_fullsized":"2024-09-13T14:56:30.472116+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":579,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512474+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:56:19.512474+0000","last_peered":"2024-09-13T14:56:19.512474+0000","last_clean":"2024-09-13T14:56:19.512474+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:56:19.512474+0000","last_undegraded":"2024-09-13T14:56:19.512474+0000","last_fullsized":"2024-09-13T14:56:19.512474+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":417,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.473958+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:56:30.473958+0000","last_peered":"2024-09-13T14:56:30.473958+0000","last_clean":"2024-09-13T14:56:30.473958+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:56:30.473958+0000","last_undegraded":"2024-09-13T14:56:30.473958+0000","last_fullsized":"2024-09-13T14:56:30.473958+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":117,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459134+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:56:30.459134+0000","last_peered":"2024-09-13T14:56:30.459134+0000","last_clean":"2024-09-13T14:56:30.459134+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:56:30.459134+0000","last_undegraded":"2024-09-13T14:56:30.459134+0000","last_fullsized":"2024-09-13T14:56:30.459134+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":110,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.467179+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:56:30.467179+0000","last_peered":"2024-09-13T14:56:30.467179+0000","last_clean":"2024-09-13T14:56:30.467179+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:56:30.467179+0000","last_undegraded":"2024-09-13T14:56:30.467179+0000","last_fullsized":"2024-09-13T14:56:30.467179+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":438,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453435+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:56:30.453435+0000","last_peered":"2024-09-13T14:56:30.453435+0000","last_clean":"2024-09-13T14:56:30.453435+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:56:30.453435+0000","last_undegraded":"2024-09-13T14:56:30.453435+0000","last_fullsized":"2024-09-13T14:56:30.453435+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":428,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.573404+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:56:30.573404+0000","last_peered":"2024-09-13T14:56:30.573404+0000","last_clean":"2024-09-13T14:56:30.573404+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:56:30.573404+0000","last_undegraded":"2024-09-13T14:56:30.573404+0000","last_fullsized":"2024-09-13T14:56:30.573404+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":425,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.477483+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:56:30.477483+0000","last_peered":"2024-09-13T14:56:30.477483+0000","last_clean":"2024-09-13T14:56:30.477483+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:56:30.477483+0000","last_undegraded":"2024-09-13T14:56:30.477483+0000","last_fullsized":"2024-09-13T14:56:30.477483+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":432,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.593169+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:56:30.593169+0000","last_peered":"2024-09-13T14:56:30.593169+0000","last_clean":"2024-09-13T14:56:30.593169+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:56:30.593169+0000","last_undegraded":"2024-09-13T14:56:30.593169+0000","last_fullsized":"2024-09-13T14:56:30.593169+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":678,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.572451+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:56:30.572451+0000","last_peered":"2024-09-13T14:56:30.572451+0000","last_clean":"2024-09-13T14:56:30.572451+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:56:30.572451+0000","last_undegraded":"2024-09-13T14:56:30.572451+0000","last_fullsized":"2024-09-13T14:56:30.572451+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":34,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459306+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:56:30.459306+0000","last_peered":"2024-09-13T14:56:30.459306+0000","last_clean":"2024-09-13T14:56:30.459306+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:56:30.459306+0000","last_undegraded":"2024-09-13T14:56:30.459306+0000","last_fullsized":"2024-09-13T14:56:30.459306+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":282,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:21.571711+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:56:21.571711+0000","last_peered":"2024-09-13T14:56:21.571711+0000","last_clean":"2024-09-13T14:56:21.571711+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:56:21.571711+0000","last_undegraded":"2024-09-13T14:56:21.571711+0000","last_fullsized":"2024-09-13T14:56:21.571711+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":41,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589667+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:56:18.589667+0000","last_peered":"2024-09-13T14:56:18.589667+0000","last_clean":"2024-09-13T14:56:18.589667+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:56:18.589667+0000","last_undegraded":"2024-09-13T14:56:18.589667+0000","last_fullsized":"2024-09-13T14:56:18.589667+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":102,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512560+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:56:19.512560+0000","last_peered":"2024-09-13T14:56:19.512560+0000","last_clean":"2024-09-13T14:56:19.512560+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:56:19.512560+0000","last_undegraded":"2024-09-13T14:56:19.512560+0000","last_fullsized":"2024-09-13T14:56:19.512560+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":100,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512370+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:56:19.512370+0000","last_peered":"2024-09-13T14:56:19.512370+0000","last_clean":"2024-09-13T14:56:19.512370+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:56:19.512370+0000","last_undegraded":"2024-09-13T14:56:19.512370+0000","last_fullsized":"2024-09-13T14:56:19.512370+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":338,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.472303+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:56:30.472303+0000","last_peered":"2024-09-13T14:56:30.472303+0000","last_clean":"2024-09-13T14:56:30.472303+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:56:30.472303+0000","last_undegraded":"2024-09-13T14:56:30.472303+0000","last_fullsized":"2024-09-13T14:56:30.472303+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":376,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.455662+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:56:30.455662+0000","last_peered":"2024-09-13T14:56:30.455662+0000","last_clean":"2024-09-13T14:56:30.455662+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:56:30.455662+0000","last_undegraded":"2024-09-13T14:56:30.455662+0000","last_fullsized":"2024-09-13T14:56:30.455662+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":266,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.590244+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:56:18.590244+0000","last_peered":"2024-09-13T14:56:18.590244+0000","last_clean":"2024-09-13T14:56:18.590244+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:56:18.590244+0000","last_undegraded":"2024-09-13T14:56:18.590244+0000","last_fullsized":"2024-09-13T14:56:18.590244+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":404,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.576779+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:56:30.576779+0000","last_peered":"2024-09-13T14:56:30.576779+0000","last_clean":"2024-09-13T14:56:30.576779+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:56:30.576779+0000","last_undegraded":"2024-09-13T14:56:30.576779+0000","last_fullsized":"2024-09-13T14:56:30.576779+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.459313+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:56:30.459313+0000","last_peered":"2024-09-13T14:56:30.459313+0000","last_clean":"2024-09-13T14:56:30.459313+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:56:30.459313+0000","last_undegraded":"2024-09-13T14:56:30.459313+0000","last_fullsized":"2024-09-13T14:56:30.459313+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":417,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.593041+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:56:30.593041+0000","last_peered":"2024-09-13T14:56:30.593041+0000","last_clean":"2024-09-13T14:56:30.593041+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:56:30.593041+0000","last_undegraded":"2024-09-13T14:56:30.593041+0000","last_fullsized":"2024-09-13T14:56:30.593041+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":269,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512609+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:56:19.512609+0000","last_peered":"2024-09-13T14:56:19.512609+0000","last_clean":"2024-09-13T14:56:19.512609+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:56:19.512609+0000","last_undegraded":"2024-09-13T14:56:19.512609+0000","last_fullsized":"2024-09-13T14:56:19.512609+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":461,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.593213+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:56:30.593213+0000","last_peered":"2024-09-13T14:56:30.593213+0000","last_clean":"2024-09-13T14:56:30.593213+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:56:30.593213+0000","last_undegraded":"2024-09-13T14:56:30.593213+0000","last_fullsized":"2024-09-13T14:56:30.593213+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":110,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.463480+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:56:30.463480+0000","last_peered":"2024-09-13T14:56:30.463480+0000","last_clean":"2024-09-13T14:56:30.463480+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:56:30.463480+0000","last_undegraded":"2024-09-13T14:56:30.463480+0000","last_fullsized":"2024-09-13T14:56:30.463480+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":423,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.472591+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:56:30.472591+0000","last_peered":"2024-09-13T14:56:30.472591+0000","last_clean":"2024-09-13T14:56:30.472591+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:56:30.472591+0000","last_undegraded":"2024-09-13T14:56:30.472591+0000","last_fullsized":"2024-09-13T14:56:30.472591+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":424,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.572838+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:56:30.572838+0000","last_peered":"2024-09-13T14:56:30.572838+0000","last_clean":"2024-09-13T14:56:30.572838+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:56:30.572838+0000","last_undegraded":"2024-09-13T14:56:30.572838+0000","last_fullsized":"2024-09-13T14:56:30.572838+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":82,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.471989+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:56:30.471989+0000","last_peered":"2024-09-13T14:56:30.471989+0000","last_clean":"2024-09-13T14:56:30.471989+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:56:30.471989+0000","last_undegraded":"2024-09-13T14:56:30.471989+0000","last_fullsized":"2024-09-13T14:56:30.471989+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":443,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.453973+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:56:30.453973+0000","last_peered":"2024-09-13T14:56:30.453973+0000","last_clean":"2024-09-13T14:56:30.453973+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:56:30.453973+0000","last_undegraded":"2024-09-13T14:56:30.453973+0000","last_fullsized":"2024-09-13T14:56:30.453973+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":30,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.466183+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:56:30.466183+0000","last_peered":"2024-09-13T14:56:30.466183+0000","last_clean":"2024-09-13T14:56:30.466183+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:56:30.466183+0000","last_undegraded":"2024-09-13T14:56:30.466183+0000","last_fullsized":"2024-09-13T14:56:30.466183+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":168,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.573512+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:56:30.573512+0000","last_peered":"2024-09-13T14:56:30.573512+0000","last_clean":"2024-09-13T14:56:30.573512+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:56:30.573512+0000","last_undegraded":"2024-09-13T14:56:30.573512+0000","last_fullsized":"2024-09-13T14:56:30.573512+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":281,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.589943+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:56:18.589943+0000","last_peered":"2024-09-13T14:56:18.589943+0000","last_clean":"2024-09-13T14:56:18.589943+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:56:18.589943+0000","last_undegraded":"2024-09-13T14:56:18.589943+0000","last_fullsized":"2024-09-13T14:56:18.589943+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":306,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.575493+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:56:30.575493+0000","last_peered":"2024-09-13T14:56:30.575493+0000","last_clean":"2024-09-13T14:56:30.575493+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:56:30.575493+0000","last_undegraded":"2024-09-13T14:56:30.575493+0000","last_fullsized":"2024-09-13T14:56:30.575493+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":266,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:18.590152+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:56:18.590152+0000","last_peered":"2024-09-13T14:56:18.590152+0000","last_clean":"2024-09-13T14:56:18.590152+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:56:18.590152+0000","last_undegraded":"2024-09-13T14:56:18.590152+0000","last_fullsized":"2024-09-13T14:56:18.590152+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":463,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.579656+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:56:30.579656+0000","last_peered":"2024-09-13T14:56:30.579656+0000","last_clean":"2024-09-13T14:56:30.579656+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:56:30.579656+0000","last_undegraded":"2024-09-13T14:56:30.579656+0000","last_fullsized":"2024-09-13T14:56:30.579656+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":440,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.572382+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:56:30.572382+0000","last_peered":"2024-09-13T14:56:30.572382+0000","last_clean":"2024-09-13T14:56:30.572382+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:56:30.572382+0000","last_undegraded":"2024-09-13T14:56:30.572382+0000","last_fullsized":"2024-09-13T14:56:30.572382+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":296,"state":"active+clean","last_fresh":"2024-09-13T14:56:19.512182+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:56:19.512182+0000","last_peered":"2024-09-13T14:56:19.512182+0000","last_clean":"2024-09-13T14:56:19.512182+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:56:19.512182+0000","last_undegraded":"2024-09-13T14:56:19.512182+0000","last_fullsized":"2024-09-13T14:56:19.512182+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":36,"reported_epoch":297,"state":"active+clean","last_fresh":"2024-09-13T14:56:30.458557+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:56:30.458557+0000","last_peered":"2024-09-13T14:56:30.458557+0000","last_clean":"2024-09-13T14:56:30.458557+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:56:30.458557+0000","last_undegraded":"2024-09-13T14:56:30.458557+0000","last_fullsized":"2024-09-13T14:56:30.458557+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234438,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991772,"kb_used_data":4112,"kb_used_omap":1,"kb_used_meta":987646,"kb_avail":92753380,"statfs":{"total":95995035648,"available":94979461120,"internally_reserved":0,"allocated":4210688,"data_stored":2573419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1011349963},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496076,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923356,"kb_used_data":3792,"kb_used_omap":1,"kb_used_meta":919550,"kb_avail":92821796,"statfs":{"total":95995035648,"available":95049519104,"internally_reserved":0,"allocated":3883008,"data_stored":2245736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":941619655},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757714,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1090448,"kb_used_data":4356,"kb_used_omap":1,"kb_used_meta":1086078,"kb_avail":92654704,"statfs":{"total":95995035648,"available":94878416896,"internally_reserved":0,"allocated":4460544,"data_stored":2831565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1112144326},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019350,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650562,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":589708,"kb_used_data":4352,"kb_used_omap":4,"kb_used_meta":585339,"kb_avail":93155444,"statfs":{"total":95995035648,"available":95391174656,"internally_reserved":0,"allocated":4456448,"data_stored":2831327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":599387695},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879497,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462496,"kb_used_data":4052,"kb_used_omap":6,"kb_used_meta":458425,"kb_avail":93282656,"statfs":{"total":95995035648,"available":95521439744,"internally_reserved":0,"allocated":4149248,"data_stored":2509019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":469427769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435513,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807812,"kb_used_data":3768,"kb_used_omap":5,"kb_used_meta":804026,"kb_avail":92937340,"statfs":{"total":95995035648,"available":95167836160,"internally_reserved":0,"allocated":3858432,"data_stored":2240364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":823322977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":33,"apply_latency_ms":33,"commit_latency_ns":33000000,"apply_latency_ns":33000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926088,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":424080,"kb_used_data":3780,"kb_used_omap":5,"kb_used_meta":420282,"kb_avail":93321072,"statfs":{"total":95995035648,"available":95560777728,"internally_reserved":0,"allocated":3870720,"data_stored":2241563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":430368938},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:56:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:41.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:41.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:41.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:41.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:41.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:41.922 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:41 smithi042 ceph-mon[113926]: from='client.35598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:41.922 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:41 smithi042 ceph-mon[113926]: pgmap v1065: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:41.922 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:41 smithi042 ceph-mon[108570]: from='client.35598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:41.922 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:41 smithi042 ceph-mon[108570]: pgmap v1065: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:41.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:41.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:41.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:41 smithi159 ceph-mon[102519]: from='client.35598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:42.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:41 smithi159 ceph-mon[102519]: pgmap v1065: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:42.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.190 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:42.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:42.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:42.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:42.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:42.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:42.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:42.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:42.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:42.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:42.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:42.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:42.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:42.805 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_7' created 2024-09-13T14:56:42.808 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:56:42.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:42.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T14:56:42.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:42.954 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:42.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:42.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4138592317' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:42.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:42 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:42.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2948991335' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:42.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:42 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:43.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4138592317' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:43.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2948991335' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4138592317' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2948991335' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:43.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:42 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:56:43.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:43.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:43.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.562 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:43.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:43.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:43.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:43.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:43.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:43.964 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: pgmap v1066: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:43.964 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[108570]: osdmap e298: 8 total, 8 up, 7 in 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: pgmap v1066: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-13T14:56:43.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:43.966 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:43 smithi042 ceph-mon[113926]: osdmap e298: 8 total, 8 up, 7 in 2024-09-13T14:56:43.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:43.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:43.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_7 rados --yes-i-really-mean-it || true 2024-09-13T14:56:44.069 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:44.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:44.219 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: pgmap v1066: 83 pgs: 83 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_7", "pg_num": 16}]': finished 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:56:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:43 smithi159 ceph-mon[102519]: osdmap e298: 8 total, 8 up, 7 in 2024-09-13T14:56:44.321 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:44.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.445 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.467 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:44.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.568 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:44.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:44.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:44.661 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:44.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:56:44.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:44.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:44.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:44.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:44.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:44.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:44.944 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:44 smithi042 ceph-mon[113926]: osdmap e299: 8 total, 8 up, 7 in 2024-09-13T14:56:44.944 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:44 smithi042 ceph-mon[108570]: osdmap e299: 8 total, 8 up, 7 in 2024-09-13T14:56:44.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:44.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:44.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:45.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.159 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:44 smithi159 ceph-mon[102519]: osdmap e299: 8 total, 8 up, 7 in 2024-09-13T14:56:45.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:45.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:45.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:45.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:45.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:45.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:45.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:45.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:45.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:45.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:45.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:45.705 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:45.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:45.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:45.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:45.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:45.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:45.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:46.001 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[113926]: pgmap v1069: 99 pgs: 3 creating+peering, 10 unknown, 86 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:46.002 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:56:46.002 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[113926]: osdmap e300: 8 total, 8 up, 7 in 2024-09-13T14:56:46.002 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.002 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[108570]: pgmap v1069: 99 pgs: 3 creating+peering, 10 unknown, 86 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:46.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:56:46.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[108570]: osdmap e300: 8 total, 8 up, 7 in 2024-09-13T14:56:46.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.003 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:45 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.033 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:46.033 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:46.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:46.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.078 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:46.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:46.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:46.200 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:46.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:45 smithi159 ceph-mon[102519]: pgmap v1069: 99 pgs: 3 creating+peering, 10 unknown, 86 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:45 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T14:56:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:45 smithi159 ceph-mon[102519]: osdmap e300: 8 total, 8 up, 7 in 2024-09-13T14:56:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:45 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:46.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:46.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:46.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:46.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:46.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:46.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:56:46.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:46.694 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:46.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:46.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:46.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:46.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:46.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:46.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:47.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:47.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:47.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:47.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.190 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:47.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:47.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:47.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:47.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:47.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:47.560 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:47.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:47.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:47.688 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:56:47.688 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T14:56:47.688 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T14:56:47.688 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T14:56:47.688 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:56:47.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:47.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:47.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:47.912 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:47.934 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:47.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:47.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:47.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:47 smithi159 ceph-mon[102519]: pgmap v1071: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:48.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:48.056 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:48.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:47 smithi042 ceph-mon[108570]: pgmap v1071: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:47 smithi042 ceph-mon[113926]: pgmap v1071: 99 pgs: 3 creating+peering, 7 unknown, 89 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:48.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:48.180 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:48.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T14:56:48.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.304 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:48.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:48.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:48.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:48.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:48.798 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:48.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:48.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:48.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:48.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:49.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:49.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:49.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:49.171 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:49.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:49.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.296 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:49.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:49.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:49.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:49.544 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:49.646 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.669 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:49.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.737 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:56:49.806 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:56:49.806 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:56:49.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:56:49.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:56:49.806 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:56:49.807 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:56:49.808 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:56:49.808 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:56:49.808 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:56:49.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:49.831 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:49.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:49.933 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:49.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:49.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:50.079 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:49 smithi042 ceph-mon[108570]: pgmap v1072: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1831618964' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:49 smithi042 ceph-mon[113926]: pgmap v1072: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1831618964' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:49] "GET /metrics HTTP/1.1" 200 34044 "" "Prometheus/2.51.0" 2024-09-13T14:56:50.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:49 smithi159 ceph-mon[102519]: pgmap v1072: 99 pgs: 3 creating+peering, 96 active+clean; 582 KiB data, 5.0 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1831618964' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:56:50.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:50.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:50.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:50.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:50.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:50.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:50.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:50.809 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:50.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:50.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:50.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:51.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:51.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:51.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:51.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:51.335 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:51.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:51.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:51.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:51.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:51.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:51.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:51.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:51.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:51.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:51.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:51.816 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_7' 2024-09-13T14:56:51.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:51.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:51.915 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:51.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:51.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:51.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:51.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:52.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:52.089 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[108570]: pgmap v1073: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[113926]: pgmap v1073: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:56:52.191 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:52.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:51 smithi159 ceph-mon[102519]: pgmap v1073: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:56:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:56:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T14:56:52.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:52.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:52.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:52.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:52.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:52.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:52.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:52.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:52.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:52.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:52.824 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.848 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:52.950 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:52.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:52.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:53.050 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:56:53.050 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:52 smithi042 ceph-mon[113926]: osdmap e301: 8 total, 8 up, 7 in 2024-09-13T14:56:53.050 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:56:53.051 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:52 smithi042 ceph-mon[108570]: osdmap e301: 8 total, 8 up, 7 in 2024-09-13T14:56:53.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:53.085 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:56:53.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:53.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:53.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3834273250' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_7", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T14:56:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:52 smithi159 ceph-mon[102519]: osdmap e301: 8 total, 8 up, 7 in 2024-09-13T14:56:53.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:53.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:53.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:53.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:53.479 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:53.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:53.581 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:53.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:53.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:53.637 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_7 bench 90 write' 2024-09-13T14:56:53.646 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T14:56:53.647 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T14:56:53.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T14:56:53.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T14:56:53.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_260592 2024-09-13T14:56:53.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:56:53.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T14:56:53.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:53.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:53.839 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:53.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:53.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:56:53.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:53.982 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[108570]: pgmap v1075: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:53.982 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:56:53.982 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T14:56:53.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[113926]: pgmap v1075: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:53.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:56:53.983 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:53 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T14:56:53.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:53.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:53.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:54.099 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:54.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:53 smithi159 ceph-mon[102519]: pgmap v1075: 99 pgs: 99 active+clean; 582 KiB data, 5.1 GiB used, 621 GiB / 626 GiB avail 2024-09-13T14:56:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:53 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T14:56:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:53 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T14:56:54.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:54.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:54.382 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:54.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:54.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:54.541 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:54.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:54.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:54.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:54.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 15 3419 3404 212.738 212.75 0.00734469 0.00468998 2024-09-13T14:56:54.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:54.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:54.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:54.958 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:54.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:54.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:55.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:55.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:55.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:55.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:55.503 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:55.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:55.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.647 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:55.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 15 6599 6584 205.735 198.75 0.00918622 0.00484694 2024-09-13T14:56:55.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:55.816 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:55.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:55.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:55.935 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:55 smithi042 ceph-mon[108570]: pgmap v1076: 99 pgs: 99 active+clean; 33 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-13T14:56:55.935 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:55 smithi042 ceph-mon[113926]: pgmap v1076: 99 pgs: 99 active+clean; 33 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-13T14:56:55.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:55.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:56.063 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:56.107 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:56.209 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.232 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:56.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:55 smithi159 ceph-mon[102519]: pgmap v1076: 99 pgs: 99 active+clean; 33 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 3.4 MiB/s wr, 53 op/s 2024-09-13T14:56:56.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:56.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:56.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:56.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:56.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:56.627 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:56.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 8980 8964 186.735 148.75 0.00306984 0.00534579 2024-09-13T14:56:56.729 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:56:56.758 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:56.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:56:56.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:56.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:56.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:57.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:57.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:57.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:56:57.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:57.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:56:57.289 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:57.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:57.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:57.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:57.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:57.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:57.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 11441 11425 178.5 153.812 0.00844996 0.00546432 2024-09-13T14:56:57.708 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:57.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:57.810 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.837 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:57.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:57.938 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:56:57.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:57.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:57.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:56:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:57 smithi159 ceph-mon[102519]: pgmap v1077: 99 pgs: 99 active+clean; 133 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 MiB/s wr, 211 op/s 2024-09-13T14:56:58.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:57 smithi042 ceph-mon[108570]: pgmap v1077: 99 pgs: 99 active+clean; 133 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 MiB/s wr, 211 op/s 2024-09-13T14:56:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:57 smithi042 ceph-mon[113926]: pgmap v1077: 99 pgs: 99 active+clean; 133 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 13 MiB/s wr, 211 op/s 2024-09-13T14:56:58.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:58.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:58.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:58.349 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:58.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:58.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:58.466 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:56:58.472 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:56:58.494 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:58.596 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:58.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:58.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:58.699 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 13546 13530 169.111 131.562 0.00625492 0.00590535 2024-09-13T14:56:58.737 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:58.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:58.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:58.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:58.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:59.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:59.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:58 smithi042 ceph-mon[113926]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:58 smithi042 ceph-mon[108570]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:59.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:56:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:59.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:56:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:58 smithi159 ceph-mon[102519]: from='client.45605 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:56:59.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:56:59.313 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:56:59.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:56:59.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:56:59.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:56:59.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:56:59.702 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 15912 15896 165.498 147.875 0.0130747 0.00603118 2024-09-13T14:56:59.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:59.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:56:59.718 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:56:59.718 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:56:59.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:56:59.841 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:56:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:56:59] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-09-13T14:56:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:56:59.861 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:56:59.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:56:59.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:00.002 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:00.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:56:59 smithi042 ceph-mon[113926]: pgmap v1078: 99 pgs: 99 active+clean; 376 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-13T14:57:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:56:59 smithi042 ceph-mon[108570]: pgmap v1078: 99 pgs: 99 active+clean; 376 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-13T14:57:00.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:00.142 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:00.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:56:59 smithi159 ceph-mon[102519]: pgmap v1078: 99 pgs: 99 active+clean; 376 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 38 MiB/s wr, 600 op/s 2024-09-13T14:57:00.289 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1077,"stamp":"2024-09-13T14:56:56.504386+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":139466725,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2183,"num_write_kb":135649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3817,"ondisk_log_size":3817,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5569788,"kb_used_data":308372,"kb_used_omap":27,"kb_used_meta":5261284,"kb_avail":650646276,"statfs":{"total":671965249536,"available":666261786624,"internally_reserved":0,"allocated":315772928,"data_stored":304267473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5387555787},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":138870784,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2119,"num_write_kb":135616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003143"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074232+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:56:52.074232+0000","last_peered":"2024-09-13T14:56:52.074232+0000","last_clean":"2024-09-13T14:56:52.074232+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:56:52.074232+0000","last_undegraded":"2024-09-13T14:56:52.074232+0000","last_fullsized":"2024-09-13T14:56:52.074232+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":92,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817711+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:56:51.817711+0000","last_peered":"2024-09-13T14:56:51.817711+0000","last_clean":"2024-09-13T14:56:51.817711+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:56:51.817711+0000","last_undegraded":"2024-09-13T14:56:51.817711+0000","last_fullsized":"2024-09-13T14:56:51.817711+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805922+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:56:43.805922+0000","last_peered":"2024-09-13T14:56:43.805922+0000","last_clean":"2024-09-13T14:56:43.805922+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:56:43.805922+0000","last_undegraded":"2024-09-13T14:56:43.805922+0000","last_fullsized":"2024-09-13T14:56:43.805922+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443186+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:56:52.443186+0000","last_peered":"2024-09-13T14:56:52.443186+0000","last_clean":"2024-09-13T14:56:52.443186+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:56:52.443186+0000","last_undegraded":"2024-09-13T14:56:52.443186+0000","last_fullsized":"2024-09-13T14:56:52.443186+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443719+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:56:52.443719+0000","last_peered":"2024-09-13T14:56:52.443719+0000","last_clean":"2024-09-13T14:56:52.443719+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:56:52.443719+0000","last_undegraded":"2024-09-13T14:56:52.443719+0000","last_fullsized":"2024-09-13T14:56:52.443719+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":44,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809362+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:56:43.809362+0000","last_peered":"2024-09-13T14:56:43.809362+0000","last_clean":"2024-09-13T14:56:43.809362+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:56:43.809362+0000","last_undegraded":"2024-09-13T14:56:43.809362+0000","last_fullsized":"2024-09-13T14:56:43.809362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":451,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074568+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:56:52.074568+0000","last_peered":"2024-09-13T14:56:52.074568+0000","last_clean":"2024-09-13T14:56:52.074568+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:56:52.074568+0000","last_undegraded":"2024-09-13T14:56:52.074568+0000","last_fullsized":"2024-09-13T14:56:52.074568+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":355,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809219+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:56:43.809219+0000","last_peered":"2024-09-13T14:56:43.809219+0000","last_clean":"2024-09-13T14:56:43.809219+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:56:43.809219+0000","last_undegraded":"2024-09-13T14:56:43.809219+0000","last_fullsized":"2024-09-13T14:56:43.809219+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"0'0","reported_seq":11,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.834615+0000","last_change":"2024-09-13T14:56:44.021228+0000","last_active":"2024-09-13T14:56:44.834615+0000","last_peered":"2024-09-13T14:56:44.834615+0000","last_clean":"2024-09-13T14:56:44.834615+0000","last_became_active":"2024-09-13T14:56:44.020880+0000","last_became_peered":"2024-09-13T14:56:44.020880+0000","last_unstale":"2024-09-13T14:56:44.834615+0000","last_undegraded":"2024-09-13T14:56:44.834615+0000","last_fullsized":"2024-09-13T14:56:44.834615+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:13.108957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":42,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443462+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:56:52.443462+0000","last_peered":"2024-09-13T14:56:52.443462+0000","last_clean":"2024-09-13T14:56:52.443462+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:56:52.443462+0000","last_undegraded":"2024-09-13T14:56:52.443462+0000","last_fullsized":"2024-09-13T14:56:52.443462+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":255,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805608+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:56:43.805608+0000","last_peered":"2024-09-13T14:56:43.805608+0000","last_clean":"2024-09-13T14:56:43.805608+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:56:43.805608+0000","last_undegraded":"2024-09-13T14:56:43.805608+0000","last_fullsized":"2024-09-13T14:56:43.805608+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"0'0","reported_seq":11,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.834591+0000","last_change":"2024-09-13T14:56:44.021313+0000","last_active":"2024-09-13T14:56:44.834591+0000","last_peered":"2024-09-13T14:56:44.834591+0000","last_clean":"2024-09-13T14:56:44.834591+0000","last_became_active":"2024-09-13T14:56:44.020980+0000","last_became_peered":"2024-09-13T14:56:44.020980+0000","last_unstale":"2024-09-13T14:56:44.834591+0000","last_undegraded":"2024-09-13T14:56:44.834591+0000","last_fullsized":"2024-09-13T14:56:44.834591+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:50.192883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":101,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818938+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:56:51.818938+0000","last_peered":"2024-09-13T14:56:51.818938+0000","last_clean":"2024-09-13T14:56:51.818938+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:56:51.818938+0000","last_undegraded":"2024-09-13T14:56:51.818938+0000","last_fullsized":"2024-09-13T14:56:51.818938+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":322,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817549+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:56:51.817549+0000","last_peered":"2024-09-13T14:56:51.817549+0000","last_clean":"2024-09-13T14:56:51.817549+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:56:51.817549+0000","last_undegraded":"2024-09-13T14:56:51.817549+0000","last_fullsized":"2024-09-13T14:56:51.817549+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.2","version":"301'511","reported_seq":528,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:56.449684+0000","last_change":"2024-09-13T14:56:44.022513+0000","last_active":"2024-09-13T14:56:56.449684+0000","last_peered":"2024-09-13T14:56:56.449684+0000","last_clean":"2024-09-13T14:56:56.449684+0000","last_became_active":"2024-09-13T14:56:44.022194+0000","last_became_peered":"2024-09-13T14:56:44.022194+0000","last_unstale":"2024-09-13T14:56:56.449684+0000","last_undegraded":"2024-09-13T14:56:56.449684+0000","last_fullsized":"2024-09-13T14:56:56.449684+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":511,"log_dups_size":0,"ondisk_log_size":511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:16.147424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33488896,"num_objects":511,"num_object_clones":0,"num_object_copies":1533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":511,"num_write_kb":32704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":172,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805941+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:56:43.805941+0000","last_peered":"2024-09-13T14:56:43.805941+0000","last_clean":"2024-09-13T14:56:43.805941+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:56:43.805941+0000","last_undegraded":"2024-09-13T14:56:43.805941+0000","last_fullsized":"2024-09-13T14:56:43.805941+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":271,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.808939+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:56:43.808939+0000","last_peered":"2024-09-13T14:56:43.808939+0000","last_clean":"2024-09-13T14:56:43.808939+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:56:43.808939+0000","last_undegraded":"2024-09-13T14:56:43.808939+0000","last_fullsized":"2024-09-13T14:56:43.808939+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"0'0","reported_seq":17,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443606+0000","last_change":"2024-09-13T14:56:44.022816+0000","last_active":"2024-09-13T14:56:52.443606+0000","last_peered":"2024-09-13T14:56:52.443606+0000","last_clean":"2024-09-13T14:56:52.443606+0000","last_became_active":"2024-09-13T14:56:44.022484+0000","last_became_peered":"2024-09-13T14:56:44.022484+0000","last_unstale":"2024-09-13T14:56:52.443606+0000","last_undegraded":"2024-09-13T14:56:52.443606+0000","last_fullsized":"2024-09-13T14:56:52.443606+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:46:25.809222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":271,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.810567+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:56:44.810567+0000","last_peered":"2024-09-13T14:56:44.810567+0000","last_clean":"2024-09-13T14:56:44.810567+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:56:44.810567+0000","last_undegraded":"2024-09-13T14:56:44.810567+0000","last_fullsized":"2024-09-13T14:56:44.810567+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443606+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:56:52.443606+0000","last_peered":"2024-09-13T14:56:52.443606+0000","last_clean":"2024-09-13T14:56:52.443606+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:56:52.443606+0000","last_undegraded":"2024-09-13T14:56:52.443606+0000","last_fullsized":"2024-09-13T14:56:52.443606+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"0'0","reported_seq":11,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.834885+0000","last_change":"2024-09-13T14:56:43.813579+0000","last_active":"2024-09-13T14:56:44.834885+0000","last_peered":"2024-09-13T14:56:44.834885+0000","last_clean":"2024-09-13T14:56:44.834885+0000","last_became_active":"2024-09-13T14:56:43.813291+0000","last_became_peered":"2024-09-13T14:56:43.813291+0000","last_unstale":"2024-09-13T14:56:44.834885+0000","last_undegraded":"2024-09-13T14:56:44.834885+0000","last_fullsized":"2024-09-13T14:56:44.834885+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:43:37.376545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":776,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.073844+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:56:52.073844+0000","last_peered":"2024-09-13T14:56:52.073844+0000","last_clean":"2024-09-13T14:56:52.073844+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:56:52.073844+0000","last_undegraded":"2024-09-13T14:56:52.073844+0000","last_fullsized":"2024-09-13T14:56:52.073844+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":469,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074428+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:56:52.074428+0000","last_peered":"2024-09-13T14:56:52.074428+0000","last_clean":"2024-09-13T14:56:52.074428+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:56:52.074428+0000","last_undegraded":"2024-09-13T14:56:52.074428+0000","last_fullsized":"2024-09-13T14:56:52.074428+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":162,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074362+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:56:52.074362+0000","last_peered":"2024-09-13T14:56:52.074362+0000","last_clean":"2024-09-13T14:56:52.074362+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:56:52.074362+0000","last_undegraded":"2024-09-13T14:56:52.074362+0000","last_fullsized":"2024-09-13T14:56:52.074362+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"0'0","reported_seq":11,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.835016+0000","last_change":"2024-09-13T14:56:43.814770+0000","last_active":"2024-09-13T14:56:44.835016+0000","last_peered":"2024-09-13T14:56:44.835016+0000","last_clean":"2024-09-13T14:56:44.835016+0000","last_became_active":"2024-09-13T14:56:43.814445+0000","last_became_peered":"2024-09-13T14:56:43.814445+0000","last_unstale":"2024-09-13T14:56:44.835016+0000","last_undegraded":"2024-09-13T14:56:44.835016+0000","last_fullsized":"2024-09-13T14:56:44.835016+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:54.365106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":540,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805448+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:56:43.805448+0000","last_peered":"2024-09-13T14:56:43.805448+0000","last_clean":"2024-09-13T14:56:43.805448+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:56:43.805448+0000","last_undegraded":"2024-09-13T14:56:43.805448+0000","last_fullsized":"2024-09-13T14:56:43.805448+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":310,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.808936+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:56:43.808936+0000","last_peered":"2024-09-13T14:56:43.808936+0000","last_clean":"2024-09-13T14:56:43.808936+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:56:43.808936+0000","last_undegraded":"2024-09-13T14:56:43.808936+0000","last_fullsized":"2024-09-13T14:56:43.808936+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.819126+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:56:51.819126+0000","last_peered":"2024-09-13T14:56:51.819126+0000","last_clean":"2024-09-13T14:56:51.819126+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:56:51.819126+0000","last_undegraded":"2024-09-13T14:56:51.819126+0000","last_fullsized":"2024-09-13T14:56:51.819126+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"301'186","reported_seq":203,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:54.483014+0000","last_change":"2024-09-13T14:56:44.023007+0000","last_active":"2024-09-13T14:56:54.483014+0000","last_peered":"2024-09-13T14:56:54.483014+0000","last_clean":"2024-09-13T14:56:54.483014+0000","last_became_active":"2024-09-13T14:56:44.022686+0000","last_became_peered":"2024-09-13T14:56:44.022686+0000","last_unstale":"2024-09-13T14:56:54.483014+0000","last_undegraded":"2024-09-13T14:56:54.483014+0000","last_fullsized":"2024-09-13T14:56:54.483014+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":186,"log_dups_size":0,"ondisk_log_size":186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:11.239050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12189696,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":186,"num_write_kb":11904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":156,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818033+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:56:51.818033+0000","last_peered":"2024-09-13T14:56:51.818033+0000","last_clean":"2024-09-13T14:56:51.818033+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:56:51.818033+0000","last_undegraded":"2024-09-13T14:56:51.818033+0000","last_fullsized":"2024-09-13T14:56:51.818033+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":132,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809295+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:56:43.809295+0000","last_peered":"2024-09-13T14:56:43.809295+0000","last_clean":"2024-09-13T14:56:43.809295+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:56:43.809295+0000","last_undegraded":"2024-09-13T14:56:43.809295+0000","last_fullsized":"2024-09-13T14:56:43.809295+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"301'167","reported_seq":184,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:54.488289+0000","last_change":"2024-09-13T14:56:44.022403+0000","last_active":"2024-09-13T14:56:54.488289+0000","last_peered":"2024-09-13T14:56:54.488289+0000","last_clean":"2024-09-13T14:56:54.488289+0000","last_became_active":"2024-09-13T14:56:44.022060+0000","last_became_peered":"2024-09-13T14:56:44.022060+0000","last_unstale":"2024-09-13T14:56:54.488289+0000","last_undegraded":"2024-09-13T14:56:54.488289+0000","last_fullsized":"2024-09-13T14:56:54.488289+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":167,"log_dups_size":0,"ondisk_log_size":167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:21.411384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10944512,"num_objects":167,"num_object_clones":0,"num_object_copies":501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":167,"num_write_kb":10688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":540,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805536+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:56:43.805536+0000","last_peered":"2024-09-13T14:56:43.805536+0000","last_clean":"2024-09-13T14:56:43.805536+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:56:43.805536+0000","last_undegraded":"2024-09-13T14:56:43.805536+0000","last_fullsized":"2024-09-13T14:56:43.805536+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":428,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074084+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:56:52.074084+0000","last_peered":"2024-09-13T14:56:52.074084+0000","last_clean":"2024-09-13T14:56:52.074084+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:56:52.074084+0000","last_undegraded":"2024-09-13T14:56:52.074084+0000","last_fullsized":"2024-09-13T14:56:52.074084+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":434,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818554+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:56:51.818554+0000","last_peered":"2024-09-13T14:56:51.818554+0000","last_clean":"2024-09-13T14:56:51.818554+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:56:51.818554+0000","last_undegraded":"2024-09-13T14:56:51.818554+0000","last_fullsized":"2024-09-13T14:56:51.818554+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"301'168","reported_seq":185,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:54.494408+0000","last_change":"2024-09-13T14:56:44.021769+0000","last_active":"2024-09-13T14:56:54.494408+0000","last_peered":"2024-09-13T14:56:54.494408+0000","last_clean":"2024-09-13T14:56:54.494408+0000","last_became_active":"2024-09-13T14:56:44.021451+0000","last_became_peered":"2024-09-13T14:56:44.021451+0000","last_unstale":"2024-09-13T14:56:54.494408+0000","last_undegraded":"2024-09-13T14:56:54.494408+0000","last_fullsized":"2024-09-13T14:56:54.494408+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":168,"log_dups_size":0,"ondisk_log_size":168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.149839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":168,"num_write_kb":10752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":502,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443080+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:56:52.443080+0000","last_peered":"2024-09-13T14:56:52.443080+0000","last_clean":"2024-09-13T14:56:52.443080+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:56:52.443080+0000","last_undegraded":"2024-09-13T14:56:52.443080+0000","last_fullsized":"2024-09-13T14:56:52.443080+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":285,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805723+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:56:43.805723+0000","last_peered":"2024-09-13T14:56:43.805723+0000","last_clean":"2024-09-13T14:56:43.805723+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:56:43.805723+0000","last_undegraded":"2024-09-13T14:56:43.805723+0000","last_fullsized":"2024-09-13T14:56:43.805723+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":413,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074023+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:56:52.074023+0000","last_peered":"2024-09-13T14:56:52.074023+0000","last_clean":"2024-09-13T14:56:52.074023+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:56:52.074023+0000","last_undegraded":"2024-09-13T14:56:52.074023+0000","last_fullsized":"2024-09-13T14:56:52.074023+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":266,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809865+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:56:43.809865+0000","last_peered":"2024-09-13T14:56:43.809865+0000","last_clean":"2024-09-13T14:56:43.809865+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:56:43.809865+0000","last_undegraded":"2024-09-13T14:56:43.809865+0000","last_fullsized":"2024-09-13T14:56:43.809865+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"0'0","reported_seq":17,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443531+0000","last_change":"2024-09-13T14:56:44.022890+0000","last_active":"2024-09-13T14:56:52.443531+0000","last_peered":"2024-09-13T14:56:52.443531+0000","last_clean":"2024-09-13T14:56:52.443531+0000","last_became_active":"2024-09-13T14:56:44.022593+0000","last_became_peered":"2024-09-13T14:56:44.022593+0000","last_unstale":"2024-09-13T14:56:52.443531+0000","last_undegraded":"2024-09-13T14:56:52.443531+0000","last_fullsized":"2024-09-13T14:56:52.443531+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:21:45.249812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":697,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817952+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:56:51.817952+0000","last_peered":"2024-09-13T14:56:51.817952+0000","last_clean":"2024-09-13T14:56:51.817952+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:56:51.817952+0000","last_undegraded":"2024-09-13T14:56:51.817952+0000","last_fullsized":"2024-09-13T14:56:51.817952+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":414,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817654+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:56:51.817654+0000","last_peered":"2024-09-13T14:56:51.817654+0000","last_clean":"2024-09-13T14:56:51.817654+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:56:51.817654+0000","last_undegraded":"2024-09-13T14:56:51.817654+0000","last_fullsized":"2024-09-13T14:56:51.817654+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443105+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:56:52.443105+0000","last_peered":"2024-09-13T14:56:52.443105+0000","last_clean":"2024-09-13T14:56:52.443105+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:56:52.443105+0000","last_undegraded":"2024-09-13T14:56:52.443105+0000","last_fullsized":"2024-09-13T14:56:52.443105+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":38,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443419+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:56:52.443419+0000","last_peered":"2024-09-13T14:56:52.443419+0000","last_clean":"2024-09-13T14:56:52.443419+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:56:52.443419+0000","last_undegraded":"2024-09-13T14:56:52.443419+0000","last_fullsized":"2024-09-13T14:56:52.443419+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"0'0","reported_seq":11,"reported_epoch":300,"state":"active+clean","last_fresh":"2024-09-13T14:56:44.835096+0000","last_change":"2024-09-13T14:56:43.814995+0000","last_active":"2024-09-13T14:56:44.835096+0000","last_peered":"2024-09-13T14:56:44.835096+0000","last_clean":"2024-09-13T14:56:44.835096+0000","last_became_active":"2024-09-13T14:56:43.814719+0000","last_became_peered":"2024-09-13T14:56:43.814719+0000","last_unstale":"2024-09-13T14:56:44.835096+0000","last_undegraded":"2024-09-13T14:56:44.835096+0000","last_fullsized":"2024-09-13T14:56:44.835096+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.360102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":615,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443239+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:56:52.443239+0000","last_peered":"2024-09-13T14:56:52.443239+0000","last_clean":"2024-09-13T14:56:52.443239+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:56:52.443239+0000","last_undegraded":"2024-09-13T14:56:52.443239+0000","last_fullsized":"2024-09-13T14:56:52.443239+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":466,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818202+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:56:51.818202+0000","last_peered":"2024-09-13T14:56:51.818202+0000","last_clean":"2024-09-13T14:56:51.818202+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:56:51.818202+0000","last_undegraded":"2024-09-13T14:56:51.818202+0000","last_fullsized":"2024-09-13T14:56:51.818202+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817755+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:56:51.817755+0000","last_peered":"2024-09-13T14:56:51.817755+0000","last_clean":"2024-09-13T14:56:51.817755+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:56:51.817755+0000","last_undegraded":"2024-09-13T14:56:51.817755+0000","last_fullsized":"2024-09-13T14:56:51.817755+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":476,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818275+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:56:51.818275+0000","last_peered":"2024-09-13T14:56:51.818275+0000","last_clean":"2024-09-13T14:56:51.818275+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:56:51.818275+0000","last_undegraded":"2024-09-13T14:56:51.818275+0000","last_fullsized":"2024-09-13T14:56:51.818275+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"0'0","reported_seq":17,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443299+0000","last_change":"2024-09-13T14:56:43.815030+0000","last_active":"2024-09-13T14:56:52.443299+0000","last_peered":"2024-09-13T14:56:52.443299+0000","last_clean":"2024-09-13T14:56:52.443299+0000","last_became_active":"2024-09-13T14:56:43.814649+0000","last_became_peered":"2024-09-13T14:56:43.814649+0000","last_unstale":"2024-09-13T14:56:52.443299+0000","last_undegraded":"2024-09-13T14:56:52.443299+0000","last_fullsized":"2024-09-13T14:56:52.443299+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:20.559574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":574,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805449+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:56:43.805449+0000","last_peered":"2024-09-13T14:56:43.805449+0000","last_clean":"2024-09-13T14:56:43.805449+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:56:43.805449+0000","last_undegraded":"2024-09-13T14:56:43.805449+0000","last_fullsized":"2024-09-13T14:56:43.805449+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":286,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805673+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:56:43.805673+0000","last_peered":"2024-09-13T14:56:43.805673+0000","last_clean":"2024-09-13T14:56:43.805673+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:56:43.805673+0000","last_undegraded":"2024-09-13T14:56:43.805673+0000","last_fullsized":"2024-09-13T14:56:43.805673+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":322,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443269+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:56:52.443269+0000","last_peered":"2024-09-13T14:56:52.443269+0000","last_clean":"2024-09-13T14:56:52.443269+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:56:52.443269+0000","last_undegraded":"2024-09-13T14:56:52.443269+0000","last_fullsized":"2024-09-13T14:56:52.443269+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":439,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818641+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:56:51.818641+0000","last_peered":"2024-09-13T14:56:51.818641+0000","last_clean":"2024-09-13T14:56:51.818641+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:56:51.818641+0000","last_undegraded":"2024-09-13T14:56:51.818641+0000","last_fullsized":"2024-09-13T14:56:51.818641+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"0'0","reported_seq":17,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817351+0000","last_change":"2024-09-13T14:56:43.813238+0000","last_active":"2024-09-13T14:56:51.817351+0000","last_peered":"2024-09-13T14:56:51.817351+0000","last_clean":"2024-09-13T14:56:51.817351+0000","last_became_active":"2024-09-13T14:56:43.812910+0000","last_became_peered":"2024-09-13T14:56:43.812910+0000","last_unstale":"2024-09-13T14:56:51.817351+0000","last_undegraded":"2024-09-13T14:56:51.817351+0000","last_fullsized":"2024-09-13T14:56:51.817351+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:33.448823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":548,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809541+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:56:43.809541+0000","last_peered":"2024-09-13T14:56:43.809541+0000","last_clean":"2024-09-13T14:56:43.809541+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:56:43.809541+0000","last_undegraded":"2024-09-13T14:56:43.809541+0000","last_fullsized":"2024-09-13T14:56:43.809541+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":287,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.806023+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:56:43.806023+0000","last_peered":"2024-09-13T14:56:43.806023+0000","last_clean":"2024-09-13T14:56:43.806023+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:56:43.806023+0000","last_undegraded":"2024-09-13T14:56:43.806023+0000","last_fullsized":"2024-09-13T14:56:43.806023+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":491,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818834+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:56:51.818834+0000","last_peered":"2024-09-13T14:56:51.818834+0000","last_clean":"2024-09-13T14:56:51.818834+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:56:51.818834+0000","last_undegraded":"2024-09-13T14:56:51.818834+0000","last_fullsized":"2024-09-13T14:56:51.818834+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":354,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.817898+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:56:51.817898+0000","last_peered":"2024-09-13T14:56:51.817898+0000","last_clean":"2024-09-13T14:56:51.817898+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:56:51.817898+0000","last_undegraded":"2024-09-13T14:56:51.817898+0000","last_fullsized":"2024-09-13T14:56:51.817898+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"301'557","reported_seq":573,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:56.444363+0000","last_change":"2024-09-13T14:56:43.815601+0000","last_active":"2024-09-13T14:56:56.444363+0000","last_peered":"2024-09-13T14:56:56.444363+0000","last_clean":"2024-09-13T14:56:56.444363+0000","last_became_active":"2024-09-13T14:56:43.815322+0000","last_became_peered":"2024-09-13T14:56:43.815322+0000","last_unstale":"2024-09-13T14:56:56.444363+0000","last_undegraded":"2024-09-13T14:56:56.444363+0000","last_fullsized":"2024-09-13T14:56:56.444363+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":557,"log_dups_size":0,"ondisk_log_size":557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:18.744443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36503552,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":557,"num_write_kb":35648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":582,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809174+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:56:43.809174+0000","last_peered":"2024-09-13T14:56:43.809174+0000","last_clean":"2024-09-13T14:56:43.809174+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:56:43.809174+0000","last_undegraded":"2024-09-13T14:56:43.809174+0000","last_fullsized":"2024-09-13T14:56:43.809174+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818731+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:56:51.818731+0000","last_peered":"2024-09-13T14:56:51.818731+0000","last_clean":"2024-09-13T14:56:51.818731+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:56:51.818731+0000","last_undegraded":"2024-09-13T14:56:51.818731+0000","last_fullsized":"2024-09-13T14:56:51.818731+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":125,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443378+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:56:52.443378+0000","last_peered":"2024-09-13T14:56:52.443378+0000","last_clean":"2024-09-13T14:56:52.443378+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:56:52.443378+0000","last_undegraded":"2024-09-13T14:56:52.443378+0000","last_fullsized":"2024-09-13T14:56:52.443378+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443378+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:56:52.443378+0000","last_peered":"2024-09-13T14:56:52.443378+0000","last_clean":"2024-09-13T14:56:52.443378+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:56:52.443378+0000","last_undegraded":"2024-09-13T14:56:52.443378+0000","last_fullsized":"2024-09-13T14:56:52.443378+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"301'530","reported_seq":546,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:56.443791+0000","last_change":"2024-09-13T14:56:43.815725+0000","last_active":"2024-09-13T14:56:56.443791+0000","last_peered":"2024-09-13T14:56:56.443791+0000","last_clean":"2024-09-13T14:56:56.443791+0000","last_became_active":"2024-09-13T14:56:43.814650+0000","last_became_peered":"2024-09-13T14:56:43.814650+0000","last_unstale":"2024-09-13T14:56:56.443791+0000","last_undegraded":"2024-09-13T14:56:56.443791+0000","last_fullsized":"2024-09-13T14:56:56.443791+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":530,"log_dups_size":0,"ondisk_log_size":530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:28.740215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34734080,"num_objects":530,"num_object_clones":0,"num_object_copies":1590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":530,"num_write_kb":33920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":446,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818781+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:56:51.818781+0000","last_peered":"2024-09-13T14:56:51.818781+0000","last_clean":"2024-09-13T14:56:51.818781+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:56:51.818781+0000","last_undegraded":"2024-09-13T14:56:51.818781+0000","last_fullsized":"2024-09-13T14:56:51.818781+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":436,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074708+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:56:52.074708+0000","last_peered":"2024-09-13T14:56:52.074708+0000","last_clean":"2024-09-13T14:56:52.074708+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:56:52.074708+0000","last_undegraded":"2024-09-13T14:56:52.074708+0000","last_fullsized":"2024-09-13T14:56:52.074708+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":433,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.819105+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:56:51.819105+0000","last_peered":"2024-09-13T14:56:51.819105+0000","last_clean":"2024-09-13T14:56:51.819105+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:56:51.819105+0000","last_undegraded":"2024-09-13T14:56:51.819105+0000","last_fullsized":"2024-09-13T14:56:51.819105+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":440,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.073884+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:56:52.073884+0000","last_peered":"2024-09-13T14:56:52.073884+0000","last_clean":"2024-09-13T14:56:52.073884+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:56:52.073884+0000","last_undegraded":"2024-09-13T14:56:52.073884+0000","last_fullsized":"2024-09-13T14:56:52.073884+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"0'0","reported_seq":17,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818613+0000","last_change":"2024-09-13T14:56:43.814035+0000","last_active":"2024-09-13T14:56:51.818613+0000","last_peered":"2024-09-13T14:56:51.818613+0000","last_clean":"2024-09-13T14:56:51.818613+0000","last_became_active":"2024-09-13T14:56:43.813694+0000","last_became_peered":"2024-09-13T14:56:43.813694+0000","last_unstale":"2024-09-13T14:56:51.818613+0000","last_undegraded":"2024-09-13T14:56:51.818613+0000","last_fullsized":"2024-09-13T14:56:51.818613+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:46.199412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":686,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074127+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:56:52.074127+0000","last_peered":"2024-09-13T14:56:52.074127+0000","last_clean":"2024-09-13T14:56:52.074127+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:56:52.074127+0000","last_undegraded":"2024-09-13T14:56:52.074127+0000","last_fullsized":"2024-09-13T14:56:52.074127+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":42,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443608+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:56:52.443608+0000","last_peered":"2024-09-13T14:56:52.443608+0000","last_clean":"2024-09-13T14:56:52.443608+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:56:52.443608+0000","last_undegraded":"2024-09-13T14:56:52.443608+0000","last_fullsized":"2024-09-13T14:56:52.443608+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":285,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809721+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:56:43.809721+0000","last_peered":"2024-09-13T14:56:43.809721+0000","last_clean":"2024-09-13T14:56:43.809721+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:56:43.809721+0000","last_undegraded":"2024-09-13T14:56:43.809721+0000","last_fullsized":"2024-09-13T14:56:43.809721+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":44,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805545+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:56:43.805545+0000","last_peered":"2024-09-13T14:56:43.805545+0000","last_clean":"2024-09-13T14:56:43.805545+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:56:43.805545+0000","last_undegraded":"2024-09-13T14:56:43.805545+0000","last_fullsized":"2024-09-13T14:56:43.805545+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.808923+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:56:43.808923+0000","last_peered":"2024-09-13T14:56:43.808923+0000","last_clean":"2024-09-13T14:56:43.808923+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:56:43.808923+0000","last_undegraded":"2024-09-13T14:56:43.808923+0000","last_fullsized":"2024-09-13T14:56:43.808923+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":103,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809525+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:56:43.809525+0000","last_peered":"2024-09-13T14:56:43.809525+0000","last_clean":"2024-09-13T14:56:43.809525+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:56:43.809525+0000","last_undegraded":"2024-09-13T14:56:43.809525+0000","last_fullsized":"2024-09-13T14:56:43.809525+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":346,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818075+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:56:51.818075+0000","last_peered":"2024-09-13T14:56:51.818075+0000","last_clean":"2024-09-13T14:56:51.818075+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:56:51.818075+0000","last_undegraded":"2024-09-13T14:56:51.818075+0000","last_fullsized":"2024-09-13T14:56:51.818075+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":384,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.819010+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:56:51.819010+0000","last_peered":"2024-09-13T14:56:51.819010+0000","last_clean":"2024-09-13T14:56:51.819010+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:56:51.819010+0000","last_undegraded":"2024-09-13T14:56:51.819010+0000","last_fullsized":"2024-09-13T14:56:51.819010+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805833+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:56:43.805833+0000","last_peered":"2024-09-13T14:56:43.805833+0000","last_clean":"2024-09-13T14:56:43.805833+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:56:43.805833+0000","last_undegraded":"2024-09-13T14:56:43.805833+0000","last_fullsized":"2024-09-13T14:56:43.805833+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":413,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.073958+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:56:52.073958+0000","last_peered":"2024-09-13T14:56:52.073958+0000","last_clean":"2024-09-13T14:56:52.073958+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:56:52.073958+0000","last_undegraded":"2024-09-13T14:56:52.073958+0000","last_fullsized":"2024-09-13T14:56:52.073958+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443718+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:56:52.443718+0000","last_peered":"2024-09-13T14:56:52.443718+0000","last_clean":"2024-09-13T14:56:52.443718+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:56:52.443718+0000","last_undegraded":"2024-09-13T14:56:52.443718+0000","last_fullsized":"2024-09-13T14:56:52.443718+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":426,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.073836+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:56:52.073836+0000","last_peered":"2024-09-13T14:56:52.073836+0000","last_clean":"2024-09-13T14:56:52.073836+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:56:52.073836+0000","last_undegraded":"2024-09-13T14:56:52.073836+0000","last_fullsized":"2024-09-13T14:56:52.073836+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.809626+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:56:43.809626+0000","last_peered":"2024-09-13T14:56:43.809626+0000","last_clean":"2024-09-13T14:56:43.809626+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:56:43.809626+0000","last_undegraded":"2024-09-13T14:56:43.809626+0000","last_fullsized":"2024-09-13T14:56:43.809626+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":469,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.073981+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:56:52.073981+0000","last_peered":"2024-09-13T14:56:52.073981+0000","last_clean":"2024-09-13T14:56:52.073981+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:56:52.073981+0000","last_undegraded":"2024-09-13T14:56:52.073981+0000","last_fullsized":"2024-09-13T14:56:52.073981+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":118,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443778+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:56:52.443778+0000","last_peered":"2024-09-13T14:56:52.443778+0000","last_clean":"2024-09-13T14:56:52.443778+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:56:52.443778+0000","last_undegraded":"2024-09-13T14:56:52.443778+0000","last_fullsized":"2024-09-13T14:56:52.443778+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":431,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818430+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:56:51.818430+0000","last_peered":"2024-09-13T14:56:51.818430+0000","last_clean":"2024-09-13T14:56:51.818430+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:56:51.818430+0000","last_undegraded":"2024-09-13T14:56:51.818430+0000","last_fullsized":"2024-09-13T14:56:51.818430+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":432,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074289+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:56:52.074289+0000","last_peered":"2024-09-13T14:56:52.074289+0000","last_clean":"2024-09-13T14:56:52.074289+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:56:52.074289+0000","last_undegraded":"2024-09-13T14:56:52.074289+0000","last_fullsized":"2024-09-13T14:56:52.074289+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":90,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.818299+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:56:51.818299+0000","last_peered":"2024-09-13T14:56:51.818299+0000","last_clean":"2024-09-13T14:56:51.818299+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:56:51.818299+0000","last_undegraded":"2024-09-13T14:56:51.818299+0000","last_fullsized":"2024-09-13T14:56:51.818299+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":451,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:51.819281+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:56:51.819281+0000","last_peered":"2024-09-13T14:56:51.819281+0000","last_clean":"2024-09-13T14:56:51.819281+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:56:51.819281+0000","last_undegraded":"2024-09-13T14:56:51.819281+0000","last_fullsized":"2024-09-13T14:56:51.819281+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":38,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443410+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:56:52.443410+0000","last_peered":"2024-09-13T14:56:52.443410+0000","last_clean":"2024-09-13T14:56:52.443410+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:56:52.443410+0000","last_undegraded":"2024-09-13T14:56:52.443410+0000","last_fullsized":"2024-09-13T14:56:52.443410+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":176,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074333+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:56:52.074333+0000","last_peered":"2024-09-13T14:56:52.074333+0000","last_clean":"2024-09-13T14:56:52.074333+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:56:52.074333+0000","last_undegraded":"2024-09-13T14:56:52.074333+0000","last_fullsized":"2024-09-13T14:56:52.074333+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":284,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.805940+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:56:43.805940+0000","last_peered":"2024-09-13T14:56:43.805940+0000","last_clean":"2024-09-13T14:56:43.805940+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:56:43.805940+0000","last_undegraded":"2024-09-13T14:56:43.805940+0000","last_fullsized":"2024-09-13T14:56:43.805940+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":314,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074232+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:56:52.074232+0000","last_peered":"2024-09-13T14:56:52.074232+0000","last_clean":"2024-09-13T14:56:52.074232+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:56:52.074232+0000","last_undegraded":"2024-09-13T14:56:52.074232+0000","last_fullsized":"2024-09-13T14:56:52.074232+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":269,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.806020+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:56:43.806020+0000","last_peered":"2024-09-13T14:56:43.806020+0000","last_clean":"2024-09-13T14:56:43.806020+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:56:43.806020+0000","last_undegraded":"2024-09-13T14:56:43.806020+0000","last_fullsized":"2024-09-13T14:56:43.806020+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":471,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074409+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:56:52.074409+0000","last_peered":"2024-09-13T14:56:52.074409+0000","last_clean":"2024-09-13T14:56:52.074409+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:56:52.074409+0000","last_undegraded":"2024-09-13T14:56:52.074409+0000","last_fullsized":"2024-09-13T14:56:52.074409+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":448,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.074609+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:56:52.074609+0000","last_peered":"2024-09-13T14:56:52.074609+0000","last_clean":"2024-09-13T14:56:52.074609+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:56:52.074609+0000","last_undegraded":"2024-09-13T14:56:52.074609+0000","last_fullsized":"2024-09-13T14:56:52.074609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":72,"reported_epoch":299,"state":"active+clean","last_fresh":"2024-09-13T14:56:43.808562+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:56:43.808562+0000","last_peered":"2024-09-13T14:56:43.808562+0000","last_clean":"2024-09-13T14:56:43.808562+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:56:43.808562+0000","last_undegraded":"2024-09-13T14:56:43.808562+0000","last_fullsized":"2024-09-13T14:56:43.808562+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":44,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.443508+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:56:52.443508+0000","last_peered":"2024-09-13T14:56:52.443508+0000","last_clean":"2024-09-13T14:56:52.443508+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:56:52.443508+0000","last_undegraded":"2024-09-13T14:56:52.443508+0000","last_fullsized":"2024-09-13T14:56:52.443508+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":138870784,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2119,"num_write_kb":135616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":365559808,"data_stored":365559808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2119,"ondisk_log_size":2119,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234441,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991812,"kb_used_data":4148,"kb_used_omap":1,"kb_used_meta":987646,"kb_avail":92753340,"statfs":{"total":95995035648,"available":94979420160,"internally_reserved":0,"allocated":4247552,"data_stored":2599073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1011349963},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496079,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":923400,"kb_used_data":3828,"kb_used_omap":1,"kb_used_meta":919550,"kb_avail":92821752,"statfs":{"total":95995035648,"available":95049474048,"internally_reserved":0,"allocated":3919872,"data_stored":2271390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":941619655},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757717,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1094588,"kb_used_data":4392,"kb_used_omap":1,"kb_used_meta":1090174,"kb_avail":92650564,"statfs":{"total":95995035648,"available":94874177536,"internally_reserved":0,"allocated":4497408,"data_stored":2857219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1116338630},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019353,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650566,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846584,"kb_used_data":269552,"kb_used_omap":4,"kb_used_meta":577019,"kb_avail":92898568,"statfs":{"total":95995035648,"available":95128133632,"internally_reserved":0,"allocated":276021248,"data_stored":274381287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":590868015},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879500,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462540,"kb_used_data":4088,"kb_used_omap":6,"kb_used_meta":458425,"kb_avail":93282612,"statfs":{"total":95995035648,"available":95521394688,"internally_reserved":0,"allocated":4186112,"data_stored":2534673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":469427769},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435517,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":822576,"kb_used_data":18536,"kb_used_omap":5,"kb_used_meta":804026,"kb_avail":92922576,"statfs":{"total":95995035648,"available":95152717824,"internally_reserved":0,"allocated":18980864,"data_stored":17347956,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":823322977},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926092,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":428288,"kb_used_data":3828,"kb_used_omap":5,"kb_used_meta":424442,"kb_avail":93316864,"statfs":{"total":95995035648,"available":95556468736,"internally_reserved":0,"allocated":3919872,"data_stored":2275875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":434628778},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":55443456,"data_stored":55443456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":310116352,"data_stored":310116352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:57:00.294 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T14:57:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.294 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:57:00.295 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_7 2024-09-13T14:57:00.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:57:00.296 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:00.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:00.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:00.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:00.462 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:00.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:00.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:00.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:00.702 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 17965 17949 160.186 128.312 0.0124668 0.00623369 2024-09-13T14:57:00.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:00.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:00.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:00.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:00.931 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:00.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:00.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:00.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:57:01.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:01.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:01.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:01.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.223 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:01.371 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:01.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:01.474 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:01.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:01.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:01.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:01.702 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 19979 19963 155.897 125.875 0.00636323 0.00640688 2024-09-13T14:57:01.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:01.775 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:01.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:01.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:01.915 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:01.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:02.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:02.044 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:02.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:01 smithi042 ceph-mon[113926]: pgmap v1079: 99 pgs: 99 active+clean; 681 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:57:02.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:01 smithi042 ceph-mon[108570]: pgmap v1079: 99 pgs: 99 active+clean; 681 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:57:02.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:02.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:02.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:01 smithi159 ceph-mon[102519]: pgmap v1079: 99 pgs: 99 active+clean; 681 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T14:57:02.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:02.320 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:02.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:02.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:02.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:02.612 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:02.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:02.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 22138 22122 153.564 134.938 0.0143526 0.00650468 2024-09-13T14:57:02.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:02.739 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:02.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:02.845 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:02.873 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:57:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:02.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:02.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:02.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:03.024 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:57:03.024 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:03.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:03.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:03.064 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:57:03.064 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:03.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1024440084' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1024440084' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.106 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:57:03.107 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:03.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:03.143 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:57:03.144 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:03.155 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:03.179 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:57:03.180 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:03.212 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:57:03.212 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:03.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:57:03.248 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:03.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1024440084' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:57:03.249 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:03.254 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:03.257 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.281 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:03.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.285 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:57:03.285 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:03.383 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:03.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.539 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:03.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:03.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:03.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:03.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 24060 24044 150.22 120.125 0.0061789 0.00665045 2024-09-13T14:57:03.780 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:03.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:03.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:03.909 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:03.932 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:03.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:03.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.034 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.074 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:04.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[108570]: pgmap v1080: 99 pgs: 99 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[108570]: osdmap e302: 8 total, 8 up, 7 in 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3222674210' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[113926]: pgmap v1080: 99 pgs: 99 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[113926]: osdmap e302: 8 total, 8 up, 7 in 2024-09-13T14:57:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3222674210' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:04.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.204 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:04.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:03 smithi159 ceph-mon[102519]: pgmap v1080: 99 pgs: 99 active+clean; 1.1 GiB data, 8.3 GiB used, 617 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:03 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:57:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:03 smithi159 ceph-mon[102519]: osdmap e302: 8 total, 8 up, 7 in 2024-09-13T14:57:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3222674210' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:04.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:04.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:04.450 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:04.490 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:04.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:04.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:04.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 25664 25648 145.677 100.25 0.00334858 0.00685747 2024-09-13T14:57:04.714 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:04.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:04.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:04.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:04.872 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:04.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:04.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:04.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.018 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.147 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:05.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:05.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:05.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:05.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 27588 27572 143.558 120.25 0.00182398 0.00695196 2024-09-13T14:57:05.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:05.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:05.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:05.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:05.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:05.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:06.039 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:06.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:06.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:05 smithi159 ceph-mon[102519]: pgmap v1082: 99 pgs: 99 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:57:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[108570]: pgmap v1082: 99 pgs: 99 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:57:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[113926]: pgmap v1082: 99 pgs: 99 active+clean; 1.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-13T14:57:06.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:06.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:06.441 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.693 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:06.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 29376 29360 141.111 111.75 0.00618091 0.00708078 2024-09-13T14:57:06.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:06.823 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:06.861 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:06.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:06.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:06.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:06.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:06.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:07.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:07.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:07.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:07.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:07.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:07.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:07.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:07.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:07.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:07.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:07.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:07.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:07.703 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 31236 31220 139.334 116.25 0.00985086 0.00717081 2024-09-13T14:57:07.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:07.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:07.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:07.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:07.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:07.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:08.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:08.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:08.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:08.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.185 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:07 smithi159 ceph-mon[102519]: pgmap v1083: 99 pgs: 99 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:57:08.286 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:08.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:08.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:08.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:07 smithi042 ceph-mon[113926]: pgmap v1083: 99 pgs: 99 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:57:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:07 smithi042 ceph-mon[108570]: pgmap v1083: 99 pgs: 99 active+clean; 1.4 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T14:57:08.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:08.444 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:08.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:08.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.582 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:08.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:08.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:08.704 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 15 32917 32902 137.045 105.125 0.00645736 0.00729178 2024-09-13T14:57:08.725 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:08.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:08.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:08.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:08.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:08.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:08.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:08.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:09.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:09.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:09.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:09.388 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:09.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:09.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:09.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:09.669 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:09.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:09.704 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 34543 34527 134.827 101.562 0.00704711 0.00740135 2024-09-13T14:57:09.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:09.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:09.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:09.916 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:09.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:09.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:09.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:57:10.011 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:09 smithi042 ceph-mon[113926]: pgmap v1084: 99 pgs: 99 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:57:10.011 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:09] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-09-13T14:57:10.011 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:09 smithi042 ceph-mon[108570]: pgmap v1084: 99 pgs: 99 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:57:10.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:10.071 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:10.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:10.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:10.200 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:10.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:09 smithi159 ceph-mon[102519]: pgmap v1084: 99 pgs: 99 active+clean; 1.7 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s 2024-09-13T14:57:10.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:10.349 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:10.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:10.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:10.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.704 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 35666 35650 131.025 70.1875 0.00187301 0.00751035 2024-09-13T14:57:10.721 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:10.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:10.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:10.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:10.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:10.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:10.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:10.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:10.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:11.123 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:11.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:11.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.369 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:11.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:11.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:11.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:11.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.690 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:11.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:11.704 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 37314 37298 129.467 103 0.00579035 0.00771851 2024-09-13T14:57:11.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:11.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:11.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:11.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:11.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:12.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:12.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:12.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:12.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.240 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:12.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:12.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:11 smithi159 ceph-mon[102519]: pgmap v1085: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:57:12.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:12.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:11 smithi042 ceph-mon[113926]: pgmap v1085: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:57:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:11 smithi042 ceph-mon[108570]: pgmap v1085: 99 pgs: 99 active+clean; 1.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 125 MiB/s wr, 1.99k op/s 2024-09-13T14:57:12.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:12.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.510 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:12.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:12.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:12.705 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 38750 38734 127.377 89.75 0.00379628 0.00778253 2024-09-13T14:57:12.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:12.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:12.858 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:12.898 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:12.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:12.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:12.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:13.021 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:13.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:13.123 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:13.149 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:13.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:13.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:13.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:13.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.427 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:13.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:13.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:13.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:13.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.695 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:13.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:57:13.704692+0000 min lat: 0.00128182 max lat: 0.340148 avg lat: 0.00797593 2024-09-13T14:57:13.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:57:13.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 40118 40102 125.283 85.5 0.00824308 0.00797593 2024-09-13T14:57:13.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:13.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:13.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:13.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:13.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:13.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:13.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:14.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:14.108 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:14.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:14.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:14.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:13 smithi159 ceph-mon[102519]: pgmap v1086: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:57:14.318 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:13 smithi042 ceph-mon[108570]: pgmap v1086: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:57:14.319 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:13 smithi042 ceph-mon[113926]: pgmap v1086: 99 pgs: 99 active+clean; 2.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T14:57:14.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:14.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:14.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:14.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:14.617 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:14.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:14.705 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 41873 41857 124.54 109.688 0.0110864 0.00802436 2024-09-13T14:57:14.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:14.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:14.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:14.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:14.938 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:14.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:15.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:15.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:15.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:15.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:15.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:15.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:15.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:15.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.479 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:15.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:15.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:15.605 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:15.705 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 43500 43484 123.501 101.688 0.0131823 0.00809237 2024-09-13T14:57:15.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:15.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:15.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:15.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:15.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:15.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:15.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:15.993 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:16.036 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:15 smithi042 ceph-mon[108570]: pgmap v1087: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:15 smithi042 ceph-mon[113926]: pgmap v1087: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:16.138 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.182 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:15 smithi159 ceph-mon[102519]: pgmap v1087: 99 pgs: 99 active+clean; 2.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 104 MiB/s wr, 1.66k op/s 2024-09-13T14:57:16.282 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:16.311 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:16.360 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:16.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:16.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:16.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.624 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:16.706 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 45098 45082 122.472 99.875 0.00583706 0.00815856 2024-09-13T14:57:16.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:16.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:16.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:16.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:16.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:16.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:17.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:17.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:17.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:17.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:17.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:17.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:17.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:17.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:17.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:17.702 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:17.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 46693 46677 121.518 99.6875 0.00647958 0.00822322 2024-09-13T14:57:17.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:17.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:17.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:17.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:17.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:17 smithi042 ceph-mon[113926]: pgmap v1088: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:57:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:17 smithi042 ceph-mon[108570]: pgmap v1088: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:57:18.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:18.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:18.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:18.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:17 smithi159 ceph-mon[102519]: pgmap v1088: 99 pgs: 99 active+clean; 2.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-13T14:57:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:18.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:18.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:18.412 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:18.514 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:18.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:18.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:18.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:18.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:18.706 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 48119 48103 120.221 89.125 0.0239251 0.00831039 2024-09-13T14:57:18.790 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:18.801 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:57:18.802 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:57:18.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:18.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:18.949 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:18.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:19.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:19.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:19.239 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:19.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:19.367 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:19.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:19.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.508 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:19.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:19.651 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:19.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 49494 49478 118.903 85.9375 0.0129566 0.00840468 2024-09-13T14:57:19.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:19.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:19.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:19.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:19.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:20.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:20.064 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:19 smithi042 ceph-mon[113926]: pgmap v1089: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:20.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:19 smithi042 ceph-mon[113926]: from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:20.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:19 smithi042 ceph-mon[108570]: pgmap v1089: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:20.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:19 smithi042 ceph-mon[108570]: from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:19] "GET /metrics HTTP/1.1" 200 36449 "" "Prometheus/2.51.0" 2024-09-13T14:57:20.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:20.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:19 smithi159 ceph-mon[102519]: pgmap v1089: 99 pgs: 99 active+clean; 2.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:19 smithi159 ceph-mon[102519]: from='client.55243 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:57:20.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:20.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:20.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:20.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:20.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:20.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:20.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 50915 50899 117.788 88.8125 0.0183121 0.00848433 2024-09-13T14:57:20.722 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:20.749 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:20.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:20.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:20.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:20.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:20.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:21.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:21.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:21.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.166 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:21.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.267 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:21.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:21.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:21.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:21.421 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50286 12.6 12:6c9a6677:::benchmark_data_smithi042_260592_object50285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.270306+0000", 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0056972940000000003, 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057277439999999999, 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50286 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270306+0000", 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270306+0000", 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270308+0000", 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.263e-06 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270330+0000", 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2761000000000001e-05 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270343+0000", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3021e-05 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.270350+0000", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0759999999999997e-06 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275754+0000", 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0054034410000000001 2024-09-13T14:57:21.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275894+0000", 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014009299999999999 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276036+0000", 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000141643 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.426 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50288 12.7 12:e66ac637:::benchmark_data_smithi042_260592_object50287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.271148+0000", 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.004855786, 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0049582150000000002, 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.427 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50288 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271148+0000", 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271148+0000", 2024-09-13T14:57:21.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271149+0000", 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2900000000000003e-07 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271168+0000", 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.967e-05 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271177+0000", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3820000000000004e-06 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271183+0000", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.596e-06 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271236+0000", 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2321999999999998e-05 2024-09-13T14:57:21.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271367+0000", 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013123 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.271499+0000", 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013178699999999999 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276079+0000", 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045799600000000001 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50290 12.6 12.32d80d66 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.272477+0000", 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0035261929999999999, 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003692692, 2024-09-13T14:57:21.432 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50290 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272477+0000", 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272477+0000", 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272478+0000", 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5600000000000004e-07 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.434 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272502+0000", 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3368999999999999e-05 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272519+0000", 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7042999999999999e-05 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272527+0000", 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2519999999999999e-06 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.435 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50293 12.7 12:ecc4a7a4:::benchmark_data_smithi042_260592_object50292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.272786+0000", 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0032174130000000001, 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034394569999999999, 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.436 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50293 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272786+0000", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272786+0000", 2024-09-13T14:57:21.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272787+0000", 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0189999999999999e-06 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272810+0000", 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3246000000000001e-05 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272823+0000", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.295e-05 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.272832+0000", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4149999999999999e-06 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273905+0000", 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010727989999999999 2024-09-13T14:57:21.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.274020+0000", 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011535 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.274145+0000", 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000125248 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45599.0:50288 12.7 e302/298)", 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.273615+0000", 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0023884800000000001, 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002672593, 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273615+0000", 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273615+0000", 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273616+0000", 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5700000000000001e-07 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273620+0000", 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1899999999999997e-06 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273630+0000", 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0199e-05 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.273637+0000", 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7429999999999996e-06 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276277+0000", 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026396430000000001 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276284+0000", 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2230000000000001e-06 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.444 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:50294 12.5 e302/298)", 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.275839+0000", 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.00016499200000000001, 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00050091000000000003, 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275839+0000", 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.445 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275839+0000", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275839+0000", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6300000000000004e-07 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.275994+0000", 2024-09-13T14:57:21.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000154776 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276007+0000", 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2774e-05 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276014+0000", 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8920000000000003e-06 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.447 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45599.0:50293 12.7 e302/298 ondisk, result = 0)", 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.276063+0000", 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9999409, 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000312658, 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276063+0000", 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276063+0000", 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276063+0000", 2024-09-13T14:57:21.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3900000000000005e-07 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276067+0000", 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.048e-06 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276077+0000", 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8689999999999999e-06 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276083+0000", 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6389999999999997e-06 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276319+0000", 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023585500000000001 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276325+0000", 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1129999999999997e-06 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276375+0000", 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0594999999999999e-05 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.452 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:50296 12.4 e302/298 12:294fa630:::benchmark_data_smithi042_260592_object50295:head v 302'3116, mlcod=302'3114)", 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.276188+0000", 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9998159, 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025917099999999999, 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276188+0000", 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276188+0000", 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.454 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276188+0000", 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1500000000000004e-07 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276266+0000", 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7367999999999995e-05 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276274+0000", 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8389999999999996e-06 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276280+0000", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.66e-06 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276383+0000", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000103219 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276403+0000", 2024-09-13T14:57:21.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9309000000000001e-05 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50297 12.7 12.85690a47 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.276347+0000", 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9996572, 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000171929, 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T14:57:21.457 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50297 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276347+0000", 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276347+0000", 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276348+0000", 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.376e-06 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276382+0000", 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3609999999999998e-05 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276393+0000", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1552999999999999e-05 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276401+0000", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7009999999999996e-06 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276522+0000", 2024-09-13T14:57:21.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012098499999999999 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:50298 12.6 12.bd911d76 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e302)", 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:20.276413+0000", 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9995904, 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164275, 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T14:57:21.461 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 50298 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276413+0000", 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:21.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276413+0000", 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276414+0000", 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4199999999999995e-07 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276435+0000", 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1273000000000001e-05 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276443+0000", 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8869999999999992e-06 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:20.276449+0000", 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0009999999999998e-06 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:57:21.464 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 10 2024-09-13T14:57:21.465 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:57:21.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:21.547 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:21.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:21.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:21.689 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:21.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 52307 52291 116.689 87 0.00817273 0.00855633 2024-09-13T14:57:21.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:21.813 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:21.915 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:21.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:21.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:22.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:22.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:22.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:21 smithi159 ceph-mon[102519]: pgmap v1090: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T14:57:22.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:21 smithi042 ceph-mon[108570]: pgmap v1090: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T14:57:22.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:21 smithi042 ceph-mon[113926]: pgmap v1090: 99 pgs: 99 active+clean; 2.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T14:57:22.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:22.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:22.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:22.607 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:22.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:22.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 53279 53263 114.76 60.75 0.00316088 0.008694 2024-09-13T14:57:22.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.864 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:22.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:22.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:22.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:22.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:23.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:23.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:23.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]} : dispatch 2024-09-13T14:57:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:22 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]} : dispatch 2024-09-13T14:57:23.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]} : dispatch 2024-09-13T14:57:23.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:22 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]} : dispatch 2024-09-13T14:57:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]} : dispatch 2024-09-13T14:57:23.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:22 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]} : dispatch 2024-09-13T14:57:23.365 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:23.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:23.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:23.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:23.670 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:23.683 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:23.683 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:23.708 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 54733 54717 113.963 90.875 0.0295407 0.00876711 2024-09-13T14:57:23.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:23.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:23.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:23.949 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:23.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:24.051 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:24.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:24.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:24.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:24.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:23 smithi159 ceph-mon[102519]: pgmap v1091: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]}]': finished 2024-09-13T14:57:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]}]': finished 2024-09-13T14:57:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:23 smithi159 ceph-mon[102519]: osdmap e303: 8 total, 8 up, 7 in 2024-09-13T14:57:24.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:24.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[113926]: pgmap v1091: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]}]': finished 2024-09-13T14:57:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]}]': finished 2024-09-13T14:57:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[113926]: osdmap e303: 8 total, 8 up, 7 in 2024-09-13T14:57:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[108570]: pgmap v1091: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T14:57:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2, 3, 5]}]': finished 2024-09-13T14:57:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.11", "id": [3, 5]}]': finished 2024-09-13T14:57:24.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:23 smithi042 ceph-mon[108570]: osdmap e303: 8 total, 8 up, 7 in 2024-09-13T14:57:24.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:24.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:24.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:24.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:24.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 56114 56098 113.071 86.3125 0.0140858 0.00883847 2024-09-13T14:57:24.721 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:24.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:24.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:24.880 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:24.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:24.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:24.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:25.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:25.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:24 smithi159 ceph-mon[102519]: osdmap e304: 8 total, 8 up, 7 in 2024-09-13T14:57:25.257 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:24 smithi042 ceph-mon[113926]: osdmap e304: 8 total, 8 up, 7 in 2024-09-13T14:57:25.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:24 smithi042 ceph-mon[108570]: osdmap e304: 8 total, 8 up, 7 in 2024-09-13T14:57:25.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:25.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.305 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1089,"stamp":"2024-09-13T14:57:18.511248+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2845251557,"num_objects":43432,"num_object_clones":0,"num_object_copies":130296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43470,"num_write_kb":2778017,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45104,"ondisk_log_size":45104,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13790608,"kb_used_data":8361176,"kb_used_omap":27,"kb_used_meta":5429348,"kb_avail":642425456,"statfs":{"total":671965249536,"available":657843666944,"internally_reserved":0,"allocated":8561844224,"data_stored":8550296459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":27701,"internal_metadata":5559653323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[11,1,11,10,12],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1314193408,"num_objects":20053,"num_object_clones":0,"num_object_copies":60159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20053,"num_write_kb":1283392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003329"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":428,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.878901+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:57:02.878901+0000","last_peered":"2024-09-13T14:57:02.878901+0000","last_clean":"2024-09-13T14:57:02.878901+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:57:02.878901+0000","last_undegraded":"2024-09-13T14:57:02.878901+0000","last_fullsized":"2024-09-13T14:57:02.878901+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":94,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.867923+0000","last_change":"2024-09-13T14:53:10.439197+0000","last_active":"2024-09-13T14:57:02.867923+0000","last_peered":"2024-09-13T14:57:02.867923+0000","last_clean":"2024-09-13T14:57:02.867923+0000","last_became_active":"2024-09-13T14:53:10.425311+0000","last_became_peered":"2024-09-13T14:53:10.425311+0000","last_unstale":"2024-09-13T14:57:02.867923+0000","last_undegraded":"2024-09-13T14:57:02.867923+0000","last_fullsized":"2024-09-13T14:57:02.867923+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:31:50.276997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":271,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:56.710446+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:56:56.710446+0000","last_peered":"2024-09-13T14:56:56.710446+0000","last_clean":"2024-09-13T14:56:56.710446+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:56:56.710446+0000","last_undegraded":"2024-09-13T14:56:56.710446+0000","last_fullsized":"2024-09-13T14:56:56.710446+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865158+0000","last_change":"2024-09-13T14:51:20.419099+0000","last_active":"2024-09-13T14:57:02.865158+0000","last_peered":"2024-09-13T14:57:02.865158+0000","last_clean":"2024-09-13T14:57:02.865158+0000","last_became_active":"2024-09-13T14:51:20.418754+0000","last_became_peered":"2024-09-13T14:51:20.418754+0000","last_unstale":"2024-09-13T14:57:02.865158+0000","last_undegraded":"2024-09-13T14:57:02.865158+0000","last_fullsized":"2024-09-13T14:57:02.865158+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:22.441339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865758+0000","last_change":"2024-09-13T14:51:20.403070+0000","last_active":"2024-09-13T14:57:02.865758+0000","last_peered":"2024-09-13T14:57:02.865758+0000","last_clean":"2024-09-13T14:57:02.865758+0000","last_became_active":"2024-09-13T14:51:20.402701+0000","last_became_peered":"2024-09-13T14:51:20.402701+0000","last_unstale":"2024-09-13T14:57:02.865758+0000","last_undegraded":"2024-09-13T14:57:02.865758+0000","last_fullsized":"2024-09-13T14:57:02.865758+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:15.569018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597430+0000","last_change":"2024-09-13T14:53:10.426480+0000","last_active":"2024-09-13T14:56:52.597430+0000","last_peered":"2024-09-13T14:56:52.597430+0000","last_clean":"2024-09-13T14:56:52.597430+0000","last_became_active":"2024-09-13T14:53:10.426019+0000","last_became_peered":"2024-09-13T14:53:10.426019+0000","last_unstale":"2024-09-13T14:56:52.597430+0000","last_undegraded":"2024-09-13T14:56:52.597430+0000","last_fullsized":"2024-09-13T14:56:52.597430+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:34:36.205248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":453,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879239+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:57:02.879239+0000","last_peered":"2024-09-13T14:57:02.879239+0000","last_clean":"2024-09-13T14:57:02.879239+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:57:02.879239+0000","last_undegraded":"2024-09-13T14:57:02.879239+0000","last_fullsized":"2024-09-13T14:57:02.879239+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":357,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597057+0000","last_change":"2024-09-13T14:54:59.235314+0000","last_active":"2024-09-13T14:56:52.597057+0000","last_peered":"2024-09-13T14:56:52.597057+0000","last_clean":"2024-09-13T14:56:52.597057+0000","last_became_active":"2024-09-13T14:54:59.227619+0000","last_became_peered":"2024-09-13T14:54:59.227619+0000","last_unstale":"2024-09-13T14:56:52.597057+0000","last_undegraded":"2024-09-13T14:56:52.597057+0000","last_fullsized":"2024-09-13T14:56:52.597057+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:49.276865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"302'2750","reported_seq":2763,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.515620+0000","last_change":"2024-09-13T14:56:44.021228+0000","last_active":"2024-09-13T14:57:16.515620+0000","last_peered":"2024-09-13T14:57:16.515620+0000","last_clean":"2024-09-13T14:57:16.515620+0000","last_became_active":"2024-09-13T14:56:44.020880+0000","last_became_peered":"2024-09-13T14:56:44.020880+0000","last_unstale":"2024-09-13T14:57:16.515620+0000","last_undegraded":"2024-09-13T14:57:16.515620+0000","last_fullsized":"2024-09-13T14:57:16.515620+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2750,"log_dups_size":0,"ondisk_log_size":2750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:21:13.108957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180224000,"num_objects":2750,"num_object_clones":0,"num_object_copies":8250,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2750,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2750,"num_write_kb":176000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":44,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865751+0000","last_change":"2024-09-13T14:54:59.239630+0000","last_active":"2024-09-13T14:57:02.865751+0000","last_peered":"2024-09-13T14:57:02.865751+0000","last_clean":"2024-09-13T14:57:02.865751+0000","last_became_active":"2024-09-13T14:54:59.233507+0000","last_became_peered":"2024-09-13T14:54:59.233507+0000","last_unstale":"2024-09-13T14:57:02.865751+0000","last_undegraded":"2024-09-13T14:57:02.865751+0000","last_fullsized":"2024-09-13T14:57:02.865751+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:50:08.798223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.710935+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:56:52.710935+0000","last_peered":"2024-09-13T14:56:52.710935+0000","last_clean":"2024-09-13T14:56:52.710935+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:56:52.710935+0000","last_undegraded":"2024-09-13T14:56:52.710935+0000","last_fullsized":"2024-09-13T14:56:52.710935+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"302'2892","reported_seq":2905,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.558937+0000","last_change":"2024-09-13T14:56:44.021313+0000","last_active":"2024-09-13T14:57:16.558937+0000","last_peered":"2024-09-13T14:57:16.558937+0000","last_clean":"2024-09-13T14:57:16.558937+0000","last_became_active":"2024-09-13T14:56:44.020980+0000","last_became_peered":"2024-09-13T14:56:44.020980+0000","last_unstale":"2024-09-13T14:57:16.558937+0000","last_undegraded":"2024-09-13T14:57:16.558937+0000","last_fullsized":"2024-09-13T14:57:16.558937+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2892,"log_dups_size":0,"ondisk_log_size":2892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:42:50.192883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189530112,"num_objects":2892,"num_object_clones":0,"num_object_copies":8676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2892,"num_write_kb":185088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":103,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865792+0000","last_change":"2024-09-13T14:54:59.249130+0000","last_active":"2024-09-13T14:57:02.865792+0000","last_peered":"2024-09-13T14:57:02.865792+0000","last_clean":"2024-09-13T14:57:02.865792+0000","last_became_active":"2024-09-13T14:54:59.235069+0000","last_became_peered":"2024-09-13T14:54:59.235069+0000","last_unstale":"2024-09-13T14:57:02.865792+0000","last_undegraded":"2024-09-13T14:57:02.865792+0000","last_fullsized":"2024-09-13T14:57:02.865792+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:46:17.944884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":324,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.868677+0000","last_change":"2024-09-13T14:53:10.418555+0000","last_active":"2024-09-13T14:57:02.868677+0000","last_peered":"2024-09-13T14:57:02.868677+0000","last_clean":"2024-09-13T14:57:02.868677+0000","last_became_active":"2024-09-13T14:53:10.418169+0000","last_became_peered":"2024-09-13T14:53:10.418169+0000","last_unstale":"2024-09-13T14:57:02.868677+0000","last_undegraded":"2024-09-13T14:57:02.868677+0000","last_fullsized":"2024-09-13T14:57:02.868677+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:20.669534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.2","version":"302'2679","reported_seq":2698,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.478324+0000","last_change":"2024-09-13T14:56:44.022513+0000","last_active":"2024-09-13T14:57:16.478324+0000","last_peered":"2024-09-13T14:57:16.478324+0000","last_clean":"2024-09-13T14:57:16.478324+0000","last_became_active":"2024-09-13T14:56:44.022194+0000","last_became_peered":"2024-09-13T14:56:44.022194+0000","last_unstale":"2024-09-13T14:57:16.478324+0000","last_undegraded":"2024-09-13T14:57:16.478324+0000","last_fullsized":"2024-09-13T14:57:16.478324+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2679,"log_dups_size":0,"ondisk_log_size":2679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:16.147424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175570944,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2679,"num_write_kb":171456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":175,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.872391+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:57:02.872391+0000","last_peered":"2024-09-13T14:57:02.872391+0000","last_clean":"2024-09-13T14:57:02.872391+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:57:02.872391+0000","last_undegraded":"2024-09-13T14:57:02.872391+0000","last_fullsized":"2024-09-13T14:57:02.872391+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597270+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:56:52.597270+0000","last_peered":"2024-09-13T14:56:52.597270+0000","last_clean":"2024-09-13T14:56:52.597270+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:56:52.597270+0000","last_undegraded":"2024-09-13T14:56:52.597270+0000","last_fullsized":"2024-09-13T14:56:52.597270+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"302'2501","reported_seq":2520,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:13.623038+0000","last_change":"2024-09-13T14:56:44.022816+0000","last_active":"2024-09-13T14:57:13.623038+0000","last_peered":"2024-09-13T14:57:13.623038+0000","last_clean":"2024-09-13T14:57:13.623038+0000","last_became_active":"2024-09-13T14:56:44.022484+0000","last_became_peered":"2024-09-13T14:56:44.022484+0000","last_unstale":"2024-09-13T14:57:13.623038+0000","last_undegraded":"2024-09-13T14:57:13.623038+0000","last_fullsized":"2024-09-13T14:57:13.623038+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2501,"log_dups_size":0,"ondisk_log_size":2501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:46:25.809222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163905536,"num_objects":2501,"num_object_clones":0,"num_object_copies":7503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2501,"num_write_kb":160064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":273,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.711361+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:56:52.711361+0000","last_peered":"2024-09-13T14:56:52.711361+0000","last_clean":"2024-09-13T14:56:52.711361+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:56:52.711361+0000","last_undegraded":"2024-09-13T14:56:52.711361+0000","last_fullsized":"2024-09-13T14:56:52.711361+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":127,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864765+0000","last_change":"2024-09-13T14:53:10.429771+0000","last_active":"2024-09-13T14:57:02.864765+0000","last_peered":"2024-09-13T14:57:02.864765+0000","last_clean":"2024-09-13T14:57:02.864765+0000","last_became_active":"2024-09-13T14:53:10.429434+0000","last_became_peered":"2024-09-13T14:53:10.429434+0000","last_unstale":"2024-09-13T14:57:02.864765+0000","last_undegraded":"2024-09-13T14:57:02.864765+0000","last_fullsized":"2024-09-13T14:57:02.864765+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:04:14.305865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.4","version":"302'2755","reported_seq":2768,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.532214+0000","last_change":"2024-09-13T14:56:43.813579+0000","last_active":"2024-09-13T14:57:16.532214+0000","last_peered":"2024-09-13T14:57:16.532214+0000","last_clean":"2024-09-13T14:57:16.532214+0000","last_became_active":"2024-09-13T14:56:43.813291+0000","last_became_peered":"2024-09-13T14:56:43.813291+0000","last_unstale":"2024-09-13T14:57:16.532214+0000","last_undegraded":"2024-09-13T14:57:16.532214+0000","last_fullsized":"2024-09-13T14:57:16.532214+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2755,"log_dups_size":0,"ondisk_log_size":2755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:43:37.376545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180551680,"num_objects":2755,"num_object_clones":0,"num_object_copies":8265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2755,"num_write_kb":176320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":778,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.880415+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:57:02.880415+0000","last_peered":"2024-09-13T14:57:02.880415+0000","last_clean":"2024-09-13T14:57:02.880415+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:57:02.880415+0000","last_undegraded":"2024-09-13T14:57:02.880415+0000","last_fullsized":"2024-09-13T14:57:02.880415+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":471,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.886296+0000","last_change":"2024-09-13T14:54:59.245485+0000","last_active":"2024-09-13T14:57:02.886296+0000","last_peered":"2024-09-13T14:57:02.886296+0000","last_clean":"2024-09-13T14:57:02.886296+0000","last_became_active":"2024-09-13T14:54:59.230327+0000","last_became_peered":"2024-09-13T14:54:59.230327+0000","last_unstale":"2024-09-13T14:57:02.886296+0000","last_undegraded":"2024-09-13T14:57:02.886296+0000","last_fullsized":"2024-09-13T14:57:02.886296+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:18.304394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":164,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879162+0000","last_change":"2024-09-13T14:53:10.421964+0000","last_active":"2024-09-13T14:57:02.879162+0000","last_peered":"2024-09-13T14:57:02.879162+0000","last_clean":"2024-09-13T14:57:02.879162+0000","last_became_active":"2024-09-13T14:53:10.421648+0000","last_became_peered":"2024-09-13T14:53:10.421648+0000","last_unstale":"2024-09-13T14:57:02.879162+0000","last_undegraded":"2024-09-13T14:57:02.879162+0000","last_fullsized":"2024-09-13T14:57:02.879162+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:40.706668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.5","version":"302'2828","reported_seq":2841,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.562006+0000","last_change":"2024-09-13T14:56:43.814770+0000","last_active":"2024-09-13T14:57:16.562006+0000","last_peered":"2024-09-13T14:57:16.562006+0000","last_clean":"2024-09-13T14:57:16.562006+0000","last_became_active":"2024-09-13T14:56:43.814445+0000","last_became_peered":"2024-09-13T14:56:43.814445+0000","last_unstale":"2024-09-13T14:57:16.562006+0000","last_undegraded":"2024-09-13T14:57:16.562006+0000","last_fullsized":"2024-09-13T14:57:16.562006+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2828,"log_dups_size":0,"ondisk_log_size":2828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:10:54.365106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185335808,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2828,"num_write_kb":180992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":542,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.710811+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:56:52.710811+0000","last_peered":"2024-09-13T14:56:52.710811+0000","last_clean":"2024-09-13T14:56:52.710811+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:56:52.710811+0000","last_undegraded":"2024-09-13T14:56:52.710811+0000","last_fullsized":"2024-09-13T14:56:52.710811+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":312,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597425+0000","last_change":"2024-09-13T14:53:10.424659+0000","last_active":"2024-09-13T14:56:52.597425+0000","last_peered":"2024-09-13T14:56:52.597425+0000","last_clean":"2024-09-13T14:56:52.597425+0000","last_became_active":"2024-09-13T14:53:10.424009+0000","last_became_peered":"2024-09-13T14:53:10.424009+0000","last_unstale":"2024-09-13T14:56:52.597425+0000","last_undegraded":"2024-09-13T14:56:52.597425+0000","last_fullsized":"2024-09-13T14:56:52.597425+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:39:14.854291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":172,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.871136+0000","last_change":"2024-09-13T14:53:10.427290+0000","last_active":"2024-09-13T14:57:02.871136+0000","last_peered":"2024-09-13T14:57:02.871136+0000","last_clean":"2024-09-13T14:57:02.871136+0000","last_became_active":"2024-09-13T14:53:10.426401+0000","last_became_peered":"2024-09-13T14:53:10.426401+0000","last_unstale":"2024-09-13T14:57:02.871136+0000","last_undegraded":"2024-09-13T14:57:02.871136+0000","last_fullsized":"2024-09-13T14:57:02.871136+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:50:47.485727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.6","version":"302'2651","reported_seq":2670,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:14.498622+0000","last_change":"2024-09-13T14:56:44.023007+0000","last_active":"2024-09-13T14:57:14.498622+0000","last_peered":"2024-09-13T14:57:14.498622+0000","last_clean":"2024-09-13T14:57:14.498622+0000","last_became_active":"2024-09-13T14:56:44.022686+0000","last_became_peered":"2024-09-13T14:56:44.022686+0000","last_unstale":"2024-09-13T14:57:14.498622+0000","last_undegraded":"2024-09-13T14:57:14.498622+0000","last_fullsized":"2024-09-13T14:57:14.498622+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2651,"log_dups_size":0,"ondisk_log_size":2651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:11.239050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2651,"num_write_kb":169664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":158,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.869142+0000","last_change":"2024-09-13T14:53:10.421347+0000","last_active":"2024-09-13T14:57:02.869142+0000","last_peered":"2024-09-13T14:57:02.869142+0000","last_clean":"2024-09-13T14:57:02.869142+0000","last_became_active":"2024-09-13T14:53:10.421018+0000","last_became_peered":"2024-09-13T14:53:10.421018+0000","last_unstale":"2024-09-13T14:57:02.869142+0000","last_undegraded":"2024-09-13T14:57:02.869142+0000","last_fullsized":"2024-09-13T14:57:02.869142+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:48:41.717049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":134,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597490+0000","last_change":"2024-09-13T14:53:10.425492+0000","last_active":"2024-09-13T14:56:52.597490+0000","last_peered":"2024-09-13T14:56:52.597490+0000","last_clean":"2024-09-13T14:56:52.597490+0000","last_became_active":"2024-09-13T14:53:10.425190+0000","last_became_peered":"2024-09-13T14:53:10.425190+0000","last_unstale":"2024-09-13T14:56:52.597490+0000","last_undegraded":"2024-09-13T14:56:52.597490+0000","last_fullsized":"2024-09-13T14:56:52.597490+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:12.466146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.7","version":"302'2620","reported_seq":2639,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:14.498769+0000","last_change":"2024-09-13T14:56:44.022403+0000","last_active":"2024-09-13T14:57:14.498769+0000","last_peered":"2024-09-13T14:57:14.498769+0000","last_clean":"2024-09-13T14:57:14.498769+0000","last_became_active":"2024-09-13T14:56:44.022060+0000","last_became_peered":"2024-09-13T14:56:44.022060+0000","last_unstale":"2024-09-13T14:57:14.498769+0000","last_undegraded":"2024-09-13T14:57:14.498769+0000","last_fullsized":"2024-09-13T14:57:14.498769+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2620,"log_dups_size":0,"ondisk_log_size":2620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:21.411384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2620,"num_write_kb":167680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":542,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.710881+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:56:52.710881+0000","last_peered":"2024-09-13T14:56:52.710881+0000","last_clean":"2024-09-13T14:56:52.710881+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:56:52.710881+0000","last_undegraded":"2024-09-13T14:56:52.710881+0000","last_fullsized":"2024-09-13T14:56:52.710881+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":430,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879710+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:57:02.879710+0000","last_peered":"2024-09-13T14:57:02.879710+0000","last_clean":"2024-09-13T14:57:02.879710+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:57:02.879710+0000","last_undegraded":"2024-09-13T14:57:02.879710+0000","last_fullsized":"2024-09-13T14:57:02.879710+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":436,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864910+0000","last_change":"2024-09-13T14:53:10.431108+0000","last_active":"2024-09-13T14:57:02.864910+0000","last_peered":"2024-09-13T14:57:02.864910+0000","last_clean":"2024-09-13T14:57:02.864910+0000","last_became_active":"2024-09-13T14:53:10.429177+0000","last_became_peered":"2024-09-13T14:53:10.429177+0000","last_unstale":"2024-09-13T14:57:02.864910+0000","last_undegraded":"2024-09-13T14:57:02.864910+0000","last_fullsized":"2024-09-13T14:57:02.864910+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:36.599084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"302'2600","reported_seq":2619,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:14.499090+0000","last_change":"2024-09-13T14:56:44.021769+0000","last_active":"2024-09-13T14:57:14.499090+0000","last_peered":"2024-09-13T14:57:14.499090+0000","last_clean":"2024-09-13T14:57:14.499090+0000","last_became_active":"2024-09-13T14:56:44.021451+0000","last_became_peered":"2024-09-13T14:56:44.021451+0000","last_unstale":"2024-09-13T14:57:14.499090+0000","last_undegraded":"2024-09-13T14:57:14.499090+0000","last_fullsized":"2024-09-13T14:57:14.499090+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2600,"log_dups_size":0,"ondisk_log_size":2600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:48.149839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170393600,"num_objects":2600,"num_object_clones":0,"num_object_copies":7800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2600,"num_write_kb":166400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":505,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864486+0000","last_change":"2024-09-13T14:55:01.477625+0000","last_active":"2024-09-13T14:57:02.864486+0000","last_peered":"2024-09-13T14:57:02.864486+0000","last_clean":"2024-09-13T14:57:02.864486+0000","last_became_active":"2024-09-13T14:55:01.477236+0000","last_became_peered":"2024-09-13T14:55:01.477236+0000","last_unstale":"2024-09-13T14:57:02.864486+0000","last_undegraded":"2024-09-13T14:57:02.864486+0000","last_fullsized":"2024-09-13T14:57:02.864486+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:41.139767+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":288,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:57:01.710819+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:57:01.710819+0000","last_peered":"2024-09-13T14:57:01.710819+0000","last_clean":"2024-09-13T14:57:01.710819+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:57:01.710819+0000","last_undegraded":"2024-09-13T14:57:01.710819+0000","last_fullsized":"2024-09-13T14:57:01.710819+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":416,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.881620+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:57:02.881620+0000","last_peered":"2024-09-13T14:57:02.881620+0000","last_clean":"2024-09-13T14:57:02.881620+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:57:02.881620+0000","last_undegraded":"2024-09-13T14:57:02.881620+0000","last_fullsized":"2024-09-13T14:57:02.881620+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":268,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597693+0000","last_change":"2024-09-13T14:53:10.427818+0000","last_active":"2024-09-13T14:56:52.597693+0000","last_peered":"2024-09-13T14:56:52.597693+0000","last_clean":"2024-09-13T14:56:52.597693+0000","last_became_active":"2024-09-13T14:53:10.426638+0000","last_became_peered":"2024-09-13T14:53:10.426638+0000","last_unstale":"2024-09-13T14:56:52.597693+0000","last_undegraded":"2024-09-13T14:56:52.597693+0000","last_fullsized":"2024-09-13T14:56:52.597693+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:36.023888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"302'2460","reported_seq":2479,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:13.628409+0000","last_change":"2024-09-13T14:56:44.022890+0000","last_active":"2024-09-13T14:57:13.628409+0000","last_peered":"2024-09-13T14:57:13.628409+0000","last_clean":"2024-09-13T14:57:13.628409+0000","last_became_active":"2024-09-13T14:56:44.022593+0000","last_became_peered":"2024-09-13T14:56:44.022593+0000","last_unstale":"2024-09-13T14:57:13.628409+0000","last_undegraded":"2024-09-13T14:57:13.628409+0000","last_fullsized":"2024-09-13T14:57:13.628409+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2460,"log_dups_size":0,"ondisk_log_size":2460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:21:45.249812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161218560,"num_objects":2460,"num_object_clones":0,"num_object_copies":7380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2460,"num_write_kb":157440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":699,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.863644+0000","last_change":"2024-09-13T14:53:13.446139+0000","last_active":"2024-09-13T14:57:02.863644+0000","last_peered":"2024-09-13T14:57:02.863644+0000","last_clean":"2024-09-13T14:57:02.863644+0000","last_became_active":"2024-09-13T14:53:13.445271+0000","last_became_peered":"2024-09-13T14:53:13.445271+0000","last_unstale":"2024-09-13T14:57:02.863644+0000","last_undegraded":"2024-09-13T14:57:02.863644+0000","last_fullsized":"2024-09-13T14:57:02.863644+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:29:08.289609+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":416,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.861867+0000","last_change":"2024-09-13T14:51:20.398494+0000","last_active":"2024-09-13T14:57:02.861867+0000","last_peered":"2024-09-13T14:57:02.861867+0000","last_clean":"2024-09-13T14:57:02.861867+0000","last_became_active":"2024-09-13T14:51:20.397871+0000","last_became_peered":"2024-09-13T14:51:20.397871+0000","last_unstale":"2024-09-13T14:57:02.861867+0000","last_undegraded":"2024-09-13T14:57:02.861867+0000","last_fullsized":"2024-09-13T14:57:02.861867+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:11.773518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865183+0000","last_change":"2024-09-13T14:53:10.428803+0000","last_active":"2024-09-13T14:57:02.865183+0000","last_peered":"2024-09-13T14:57:02.865183+0000","last_clean":"2024-09-13T14:57:02.865183+0000","last_became_active":"2024-09-13T14:53:10.428482+0000","last_became_peered":"2024-09-13T14:53:10.428482+0000","last_unstale":"2024-09-13T14:57:02.865183+0000","last_undegraded":"2024-09-13T14:57:02.865183+0000","last_fullsized":"2024-09-13T14:57:02.865183+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:15:17.289090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":41,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865558+0000","last_change":"2024-09-13T14:54:59.236114+0000","last_active":"2024-09-13T14:57:02.865558+0000","last_peered":"2024-09-13T14:57:02.865558+0000","last_clean":"2024-09-13T14:57:02.865558+0000","last_became_active":"2024-09-13T14:54:59.235773+0000","last_became_peered":"2024-09-13T14:54:59.235773+0000","last_unstale":"2024-09-13T14:57:02.865558+0000","last_undegraded":"2024-09-13T14:57:02.865558+0000","last_fullsized":"2024-09-13T14:57:02.865558+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:16:06.400736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.f","version":"302'2835","reported_seq":2848,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.577242+0000","last_change":"2024-09-13T14:56:43.814995+0000","last_active":"2024-09-13T14:57:16.577242+0000","last_peered":"2024-09-13T14:57:16.577242+0000","last_clean":"2024-09-13T14:57:16.577242+0000","last_became_active":"2024-09-13T14:56:43.814719+0000","last_became_peered":"2024-09-13T14:56:43.814719+0000","last_unstale":"2024-09-13T14:57:16.577242+0000","last_undegraded":"2024-09-13T14:57:16.577242+0000","last_fullsized":"2024-09-13T14:57:16.577242+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2835,"log_dups_size":0,"ondisk_log_size":2835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.360102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185794560,"num_objects":2835,"num_object_clones":0,"num_object_copies":8505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2835,"num_write_kb":181440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":617,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865261+0000","last_change":"2024-09-13T14:53:13.436920+0000","last_active":"2024-09-13T14:57:02.865261+0000","last_peered":"2024-09-13T14:57:02.865261+0000","last_clean":"2024-09-13T14:57:02.865261+0000","last_became_active":"2024-09-13T14:53:13.436450+0000","last_became_peered":"2024-09-13T14:53:13.436450+0000","last_unstale":"2024-09-13T14:57:02.865261+0000","last_undegraded":"2024-09-13T14:57:02.865261+0000","last_fullsized":"2024-09-13T14:57:02.865261+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:58.930633+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":468,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.869058+0000","last_change":"2024-09-13T14:51:20.407356+0000","last_active":"2024-09-13T14:57:02.869058+0000","last_peered":"2024-09-13T14:57:02.869058+0000","last_clean":"2024-09-13T14:57:02.869058+0000","last_became_active":"2024-09-13T14:51:20.407047+0000","last_became_peered":"2024-09-13T14:51:20.407047+0000","last_unstale":"2024-09-13T14:57:02.869058+0000","last_undegraded":"2024-09-13T14:57:02.869058+0000","last_fullsized":"2024-09-13T14:57:02.869058+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:02:02.897134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":428,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.862111+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:57:02.862111+0000","last_peered":"2024-09-13T14:57:02.862111+0000","last_clean":"2024-09-13T14:57:02.862111+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:57:02.862111+0000","last_undegraded":"2024-09-13T14:57:02.862111+0000","last_fullsized":"2024-09-13T14:57:02.862111+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":478,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864683+0000","last_change":"2024-09-13T14:51:20.413225+0000","last_active":"2024-09-13T14:57:02.864683+0000","last_peered":"2024-09-13T14:57:02.864683+0000","last_clean":"2024-09-13T14:57:02.864683+0000","last_became_active":"2024-09-13T14:51:20.412838+0000","last_became_peered":"2024-09-13T14:51:20.412838+0000","last_unstale":"2024-09-13T14:57:02.864683+0000","last_undegraded":"2024-09-13T14:57:02.864683+0000","last_fullsized":"2024-09-13T14:57:02.864683+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:14:34.555703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"302'2454","reported_seq":2473,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:13.635397+0000","last_change":"2024-09-13T14:56:43.815030+0000","last_active":"2024-09-13T14:57:13.635397+0000","last_peered":"2024-09-13T14:57:13.635397+0000","last_clean":"2024-09-13T14:57:13.635397+0000","last_became_active":"2024-09-13T14:56:43.814649+0000","last_became_peered":"2024-09-13T14:56:43.814649+0000","last_unstale":"2024-09-13T14:57:13.635397+0000","last_undegraded":"2024-09-13T14:57:13.635397+0000","last_fullsized":"2024-09-13T14:57:13.635397+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2454,"log_dups_size":0,"ondisk_log_size":2454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:20.559574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160825344,"num_objects":2454,"num_object_clones":0,"num_object_copies":7362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2454,"num_write_kb":157056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":576,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.710885+0000","last_change":"2024-09-13T14:51:23.785683+0000","last_active":"2024-09-13T14:56:52.710885+0000","last_peered":"2024-09-13T14:56:52.710885+0000","last_clean":"2024-09-13T14:56:52.710885+0000","last_became_active":"2024-09-13T14:51:23.785316+0000","last_became_peered":"2024-09-13T14:51:23.785316+0000","last_unstale":"2024-09-13T14:56:52.710885+0000","last_undegraded":"2024-09-13T14:56:52.710885+0000","last_fullsized":"2024-09-13T14:56:52.710885+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:58.494972+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":289,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:57:01.710806+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:57:01.710806+0000","last_peered":"2024-09-13T14:57:01.710806+0000","last_clean":"2024-09-13T14:57:01.710806+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:57:01.710806+0000","last_undegraded":"2024-09-13T14:57:01.710806+0000","last_fullsized":"2024-09-13T14:57:01.710806+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":324,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864436+0000","last_change":"2024-09-13T14:53:10.428040+0000","last_active":"2024-09-13T14:57:02.864436+0000","last_peered":"2024-09-13T14:57:02.864436+0000","last_clean":"2024-09-13T14:57:02.864436+0000","last_became_active":"2024-09-13T14:53:10.427664+0000","last_became_peered":"2024-09-13T14:53:10.427664+0000","last_unstale":"2024-09-13T14:57:02.864436+0000","last_undegraded":"2024-09-13T14:57:02.864436+0000","last_fullsized":"2024-09-13T14:57:02.864436+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:23.969191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":441,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.862069+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:57:02.862069+0000","last_peered":"2024-09-13T14:57:02.862069+0000","last_clean":"2024-09-13T14:57:02.862069+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:57:02.862069+0000","last_undegraded":"2024-09-13T14:57:02.862069+0000","last_fullsized":"2024-09-13T14:57:02.862069+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"302'2830","reported_seq":2849,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.863260+0000","last_change":"2024-09-13T14:56:43.813238+0000","last_active":"2024-09-13T14:57:16.863260+0000","last_peered":"2024-09-13T14:57:16.863260+0000","last_clean":"2024-09-13T14:57:16.863260+0000","last_became_active":"2024-09-13T14:56:43.812910+0000","last_became_peered":"2024-09-13T14:56:43.812910+0000","last_unstale":"2024-09-13T14:57:16.863260+0000","last_undegraded":"2024-09-13T14:57:16.863260+0000","last_fullsized":"2024-09-13T14:57:16.863260+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2830,"log_dups_size":0,"ondisk_log_size":2830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:31:33.448823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185466880,"num_objects":2830,"num_object_clones":0,"num_object_copies":8490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2830,"num_write_kb":181120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":550,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597528+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:56:52.597528+0000","last_peered":"2024-09-13T14:56:52.597528+0000","last_clean":"2024-09-13T14:56:52.597528+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:56:52.597528+0000","last_undegraded":"2024-09-13T14:56:52.597528+0000","last_fullsized":"2024-09-13T14:56:52.597528+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":289,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.711398+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:56:52.711398+0000","last_peered":"2024-09-13T14:56:52.711398+0000","last_clean":"2024-09-13T14:56:52.711398+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:56:52.711398+0000","last_undegraded":"2024-09-13T14:56:52.711398+0000","last_fullsized":"2024-09-13T14:56:52.711398+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":493,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864338+0000","last_change":"2024-09-13T14:51:20.399126+0000","last_active":"2024-09-13T14:57:02.864338+0000","last_peered":"2024-09-13T14:57:02.864338+0000","last_clean":"2024-09-13T14:57:02.864338+0000","last_became_active":"2024-09-13T14:51:20.397881+0000","last_became_peered":"2024-09-13T14:51:20.397881+0000","last_unstale":"2024-09-13T14:57:02.864338+0000","last_undegraded":"2024-09-13T14:57:02.864338+0000","last_fullsized":"2024-09-13T14:57:02.864338+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:31.491199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":356,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.862019+0000","last_change":"2024-09-13T14:53:10.428298+0000","last_active":"2024-09-13T14:57:02.862019+0000","last_peered":"2024-09-13T14:57:02.862019+0000","last_clean":"2024-09-13T14:57:02.862019+0000","last_became_active":"2024-09-13T14:53:10.427980+0000","last_became_peered":"2024-09-13T14:53:10.427980+0000","last_unstale":"2024-09-13T14:57:02.862019+0000","last_undegraded":"2024-09-13T14:57:02.862019+0000","last_fullsized":"2024-09-13T14:57:02.862019+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:43:54.062599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"302'2839","reported_seq":2858,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.473759+0000","last_change":"2024-09-13T14:56:43.815601+0000","last_active":"2024-09-13T14:57:16.473759+0000","last_peered":"2024-09-13T14:57:16.473759+0000","last_clean":"2024-09-13T14:57:16.473759+0000","last_became_active":"2024-09-13T14:56:43.815322+0000","last_became_peered":"2024-09-13T14:56:43.815322+0000","last_unstale":"2024-09-13T14:57:16.473759+0000","last_undegraded":"2024-09-13T14:57:16.473759+0000","last_fullsized":"2024-09-13T14:57:16.473759+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2839,"log_dups_size":0,"ondisk_log_size":2839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:18.744443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186056704,"num_objects":2839,"num_object_clones":0,"num_object_copies":8517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2839,"num_write_kb":181696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":584,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597547+0000","last_change":"2024-09-13T14:51:23.785881+0000","last_active":"2024-09-13T14:56:52.597547+0000","last_peered":"2024-09-13T14:56:52.597547+0000","last_clean":"2024-09-13T14:56:52.597547+0000","last_became_active":"2024-09-13T14:51:23.785455+0000","last_became_peered":"2024-09-13T14:51:23.785455+0000","last_unstale":"2024-09-13T14:56:52.597547+0000","last_undegraded":"2024-09-13T14:56:52.597547+0000","last_fullsized":"2024-09-13T14:56:52.597547+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:53.236973+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":428,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864129+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:57:02.864129+0000","last_peered":"2024-09-13T14:57:02.864129+0000","last_clean":"2024-09-13T14:57:02.864129+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:57:02.864129+0000","last_undegraded":"2024-09-13T14:57:02.864129+0000","last_fullsized":"2024-09-13T14:57:02.864129+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":127,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865477+0000","last_change":"2024-09-13T14:53:10.424075+0000","last_active":"2024-09-13T14:57:02.865477+0000","last_peered":"2024-09-13T14:57:02.865477+0000","last_clean":"2024-09-13T14:57:02.865477+0000","last_became_active":"2024-09-13T14:53:10.423729+0000","last_became_peered":"2024-09-13T14:53:10.423729+0000","last_unstale":"2024-09-13T14:57:02.865477+0000","last_undegraded":"2024-09-13T14:57:02.865477+0000","last_fullsized":"2024-09-13T14:57:02.865477+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:22.588943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864530+0000","last_change":"2024-09-13T14:51:20.411369+0000","last_active":"2024-09-13T14:57:02.864530+0000","last_peered":"2024-09-13T14:57:02.864530+0000","last_clean":"2024-09-13T14:57:02.864530+0000","last_became_active":"2024-09-13T14:51:20.410832+0000","last_became_peered":"2024-09-13T14:51:20.410832+0000","last_unstale":"2024-09-13T14:57:02.864530+0000","last_undegraded":"2024-09-13T14:57:02.864530+0000","last_fullsized":"2024-09-13T14:57:02.864530+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:22:25.450640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"12.8","version":"302'2841","reported_seq":2860,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.483068+0000","last_change":"2024-09-13T14:56:43.815725+0000","last_active":"2024-09-13T14:57:16.483068+0000","last_peered":"2024-09-13T14:57:16.483068+0000","last_clean":"2024-09-13T14:57:16.483068+0000","last_became_active":"2024-09-13T14:56:43.814650+0000","last_became_peered":"2024-09-13T14:56:43.814650+0000","last_unstale":"2024-09-13T14:57:16.483068+0000","last_undegraded":"2024-09-13T14:57:16.483068+0000","last_fullsized":"2024-09-13T14:57:16.483068+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2841,"log_dups_size":0,"ondisk_log_size":2841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:23:28.740215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2841,"num_write_kb":181824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":448,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.870968+0000","last_change":"2024-09-13T14:53:14.267571+0000","last_active":"2024-09-13T14:57:02.870968+0000","last_peered":"2024-09-13T14:57:02.870968+0000","last_clean":"2024-09-13T14:57:02.870968+0000","last_became_active":"2024-09-13T14:53:13.514629+0000","last_became_peered":"2024-09-13T14:53:13.514629+0000","last_unstale":"2024-09-13T14:57:02.870968+0000","last_undegraded":"2024-09-13T14:57:02.870968+0000","last_fullsized":"2024-09-13T14:57:02.870968+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:18:29.520992+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":438,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879451+0000","last_change":"2024-09-13T14:51:20.430463+0000","last_active":"2024-09-13T14:57:02.879451+0000","last_peered":"2024-09-13T14:57:02.879451+0000","last_clean":"2024-09-13T14:57:02.879451+0000","last_became_active":"2024-09-13T14:51:20.410048+0000","last_became_peered":"2024-09-13T14:51:20.410048+0000","last_unstale":"2024-09-13T14:57:02.879451+0000","last_undegraded":"2024-09-13T14:57:02.879451+0000","last_fullsized":"2024-09-13T14:57:02.879451+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:15.766991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":435,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.866839+0000","last_change":"2024-09-13T14:53:10.430853+0000","last_active":"2024-09-13T14:57:02.866839+0000","last_peered":"2024-09-13T14:57:02.866839+0000","last_clean":"2024-09-13T14:57:02.866839+0000","last_became_active":"2024-09-13T14:53:10.429122+0000","last_became_peered":"2024-09-13T14:53:10.429122+0000","last_unstale":"2024-09-13T14:57:02.866839+0000","last_undegraded":"2024-09-13T14:57:02.866839+0000","last_fullsized":"2024-09-13T14:57:02.866839+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:40:37.500620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":442,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879601+0000","last_change":"2024-09-13T14:54:59.229050+0000","last_active":"2024-09-13T14:57:02.879601+0000","last_peered":"2024-09-13T14:57:02.879601+0000","last_clean":"2024-09-13T14:57:02.879601+0000","last_became_active":"2024-09-13T14:54:59.227349+0000","last_became_peered":"2024-09-13T14:54:59.227349+0000","last_unstale":"2024-09-13T14:57:02.879601+0000","last_undegraded":"2024-09-13T14:57:02.879601+0000","last_fullsized":"2024-09-13T14:57:02.879601+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:36.961527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"302'2871","reported_seq":2890,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:16.863273+0000","last_change":"2024-09-13T14:56:43.814035+0000","last_active":"2024-09-13T14:57:16.863273+0000","last_peered":"2024-09-13T14:57:16.863273+0000","last_clean":"2024-09-13T14:57:16.863273+0000","last_became_active":"2024-09-13T14:56:43.813694+0000","last_became_peered":"2024-09-13T14:56:43.813694+0000","last_unstale":"2024-09-13T14:57:16.863273+0000","last_undegraded":"2024-09-13T14:57:16.863273+0000","last_fullsized":"2024-09-13T14:57:16.863273+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":2871,"log_dups_size":0,"ondisk_log_size":2871,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:46.199412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188153856,"num_objects":2871,"num_object_clones":0,"num_object_copies":8613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2871,"num_write_kb":183744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":688,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.878709+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:57:02.878709+0000","last_peered":"2024-09-13T14:57:02.878709+0000","last_clean":"2024-09-13T14:57:02.878709+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:57:02.878709+0000","last_undegraded":"2024-09-13T14:57:02.878709+0000","last_fullsized":"2024-09-13T14:57:02.878709+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":45,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865710+0000","last_change":"2024-09-13T14:54:59.247600+0000","last_active":"2024-09-13T14:57:02.865710+0000","last_peered":"2024-09-13T14:57:02.865710+0000","last_clean":"2024-09-13T14:57:02.865710+0000","last_became_active":"2024-09-13T14:54:59.236510+0000","last_became_peered":"2024-09-13T14:54:59.236510+0000","last_unstale":"2024-09-13T14:57:02.865710+0000","last_undegraded":"2024-09-13T14:57:02.865710+0000","last_fullsized":"2024-09-13T14:57:02.865710+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:25:48.615006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":287,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597783+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:56:52.597783+0000","last_peered":"2024-09-13T14:56:52.597783+0000","last_clean":"2024-09-13T14:56:52.597783+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:56:52.597783+0000","last_undegraded":"2024-09-13T14:56:52.597783+0000","last_fullsized":"2024-09-13T14:56:52.597783+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.710971+0000","last_change":"2024-09-13T14:53:10.425979+0000","last_active":"2024-09-13T14:56:52.710971+0000","last_peered":"2024-09-13T14:56:52.710971+0000","last_clean":"2024-09-13T14:56:52.710971+0000","last_became_active":"2024-09-13T14:53:10.425624+0000","last_became_peered":"2024-09-13T14:53:10.425624+0000","last_unstale":"2024-09-13T14:56:52.710971+0000","last_undegraded":"2024-09-13T14:56:52.710971+0000","last_fullsized":"2024-09-13T14:56:52.710971+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:16:06.926185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":107,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597588+0000","last_change":"2024-09-13T14:53:10.429206+0000","last_active":"2024-09-13T14:56:52.597588+0000","last_peered":"2024-09-13T14:56:52.597588+0000","last_clean":"2024-09-13T14:56:52.597588+0000","last_became_active":"2024-09-13T14:53:10.428883+0000","last_became_peered":"2024-09-13T14:53:10.428883+0000","last_unstale":"2024-09-13T14:56:52.597588+0000","last_undegraded":"2024-09-13T14:56:52.597588+0000","last_fullsized":"2024-09-13T14:56:52.597588+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:37.421775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":105,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597550+0000","last_change":"2024-09-13T14:53:10.435115+0000","last_active":"2024-09-13T14:56:52.597550+0000","last_peered":"2024-09-13T14:56:52.597550+0000","last_clean":"2024-09-13T14:56:52.597550+0000","last_became_active":"2024-09-13T14:53:10.420297+0000","last_became_peered":"2024-09-13T14:53:10.420297+0000","last_unstale":"2024-09-13T14:56:52.597550+0000","last_undegraded":"2024-09-13T14:56:52.597550+0000","last_fullsized":"2024-09-13T14:56:52.597550+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:12.054334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":348,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.866102+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:57:02.866102+0000","last_peered":"2024-09-13T14:57:02.866102+0000","last_clean":"2024-09-13T14:57:02.866102+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:57:02.866102+0000","last_undegraded":"2024-09-13T14:57:02.866102+0000","last_fullsized":"2024-09-13T14:57:02.866102+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":387,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.871141+0000","last_change":"2024-09-13T14:53:10.442101+0000","last_active":"2024-09-13T14:57:02.871141+0000","last_peered":"2024-09-13T14:57:02.871141+0000","last_clean":"2024-09-13T14:57:02.871141+0000","last_became_active":"2024-09-13T14:53:10.426952+0000","last_became_peered":"2024-09-13T14:53:10.426952+0000","last_unstale":"2024-09-13T14:57:02.871141+0000","last_undegraded":"2024-09-13T14:57:02.871141+0000","last_fullsized":"2024-09-13T14:57:02.871141+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:22.427729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:57:01.710862+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:57:01.710862+0000","last_peered":"2024-09-13T14:57:01.710862+0000","last_clean":"2024-09-13T14:57:01.710862+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:57:01.710862+0000","last_undegraded":"2024-09-13T14:57:01.710862+0000","last_fullsized":"2024-09-13T14:57:01.710862+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":415,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.880502+0000","last_change":"2024-09-13T14:40:37.187003+0000","last_active":"2024-09-13T14:57:02.880502+0000","last_peered":"2024-09-13T14:57:02.880502+0000","last_clean":"2024-09-13T14:57:02.880502+0000","last_became_active":"2024-09-13T14:40:37.186601+0000","last_became_peered":"2024-09-13T14:40:37.186601+0000","last_unstale":"2024-09-13T14:57:02.880502+0000","last_undegraded":"2024-09-13T14:57:02.880502+0000","last_fullsized":"2024-09-13T14:57:02.880502+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:24:21.287958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865985+0000","last_change":"2024-09-13T14:51:20.419867+0000","last_active":"2024-09-13T14:57:02.865985+0000","last_peered":"2024-09-13T14:57:02.865985+0000","last_clean":"2024-09-13T14:57:02.865985+0000","last_became_active":"2024-09-13T14:51:20.419529+0000","last_became_peered":"2024-09-13T14:51:20.419529+0000","last_unstale":"2024-09-13T14:57:02.865985+0000","last_undegraded":"2024-09-13T14:57:02.865985+0000","last_fullsized":"2024-09-13T14:57:02.865985+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:27:19.453895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":428,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879508+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:57:02.879508+0000","last_peered":"2024-09-13T14:57:02.879508+0000","last_clean":"2024-09-13T14:57:02.879508+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:57:02.879508+0000","last_undegraded":"2024-09-13T14:57:02.879508+0000","last_fullsized":"2024-09-13T14:57:02.879508+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.597676+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:56:52.597676+0000","last_peered":"2024-09-13T14:56:52.597676+0000","last_clean":"2024-09-13T14:56:52.597676+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:56:52.597676+0000","last_undegraded":"2024-09-13T14:56:52.597676+0000","last_fullsized":"2024-09-13T14:56:52.597676+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":471,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879669+0000","last_change":"2024-09-13T14:54:59.229569+0000","last_active":"2024-09-13T14:57:02.879669+0000","last_peered":"2024-09-13T14:57:02.879669+0000","last_clean":"2024-09-13T14:57:02.879669+0000","last_became_active":"2024-09-13T14:54:59.228156+0000","last_became_peered":"2024-09-13T14:54:59.228156+0000","last_unstale":"2024-09-13T14:57:02.879669+0000","last_undegraded":"2024-09-13T14:57:02.879669+0000","last_fullsized":"2024-09-13T14:57:02.879669+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:00:43.378155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":120,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865931+0000","last_change":"2024-09-13T14:51:20.419808+0000","last_active":"2024-09-13T14:57:02.865931+0000","last_peered":"2024-09-13T14:57:02.865931+0000","last_clean":"2024-09-13T14:57:02.865931+0000","last_became_active":"2024-09-13T14:51:20.419478+0000","last_became_peered":"2024-09-13T14:51:20.419478+0000","last_unstale":"2024-09-13T14:57:02.865931+0000","last_undegraded":"2024-09-13T14:57:02.865931+0000","last_fullsized":"2024-09-13T14:57:02.865931+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:11:57.631990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":433,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865934+0000","last_change":"2024-09-13T14:53:10.422699+0000","last_active":"2024-09-13T14:57:02.865934+0000","last_peered":"2024-09-13T14:57:02.865934+0000","last_clean":"2024-09-13T14:57:02.865934+0000","last_became_active":"2024-09-13T14:53:10.422376+0000","last_became_peered":"2024-09-13T14:53:10.422376+0000","last_unstale":"2024-09-13T14:57:02.865934+0000","last_undegraded":"2024-09-13T14:57:02.865934+0000","last_fullsized":"2024-09-13T14:57:02.865934+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:31.646585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":434,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.878878+0000","last_change":"2024-09-13T14:53:10.427095+0000","last_active":"2024-09-13T14:57:02.878878+0000","last_peered":"2024-09-13T14:57:02.878878+0000","last_clean":"2024-09-13T14:57:02.878878+0000","last_became_active":"2024-09-13T14:53:10.426500+0000","last_became_peered":"2024-09-13T14:53:10.426500+0000","last_unstale":"2024-09-13T14:57:02.878878+0000","last_undegraded":"2024-09-13T14:57:02.878878+0000","last_fullsized":"2024-09-13T14:57:02.878878+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:45:44.091753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":92,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.866007+0000","last_change":"2024-09-13T14:53:10.430032+0000","last_active":"2024-09-13T14:57:02.866007+0000","last_peered":"2024-09-13T14:57:02.866007+0000","last_clean":"2024-09-13T14:57:02.866007+0000","last_became_active":"2024-09-13T14:53:10.429678+0000","last_became_peered":"2024-09-13T14:53:10.429678+0000","last_unstale":"2024-09-13T14:57:02.866007+0000","last_undegraded":"2024-09-13T14:57:02.866007+0000","last_fullsized":"2024-09-13T14:57:02.866007+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:05.037499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":453,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.870658+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:57:02.870658+0000","last_peered":"2024-09-13T14:57:02.870658+0000","last_clean":"2024-09-13T14:57:02.870658+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:57:02.870658+0000","last_undegraded":"2024-09-13T14:57:02.870658+0000","last_fullsized":"2024-09-13T14:57:02.870658+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":40,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.864330+0000","last_change":"2024-09-13T14:54:59.229600+0000","last_active":"2024-09-13T14:57:02.864330+0000","last_peered":"2024-09-13T14:57:02.864330+0000","last_clean":"2024-09-13T14:57:02.864330+0000","last_became_active":"2024-09-13T14:54:59.229250+0000","last_became_peered":"2024-09-13T14:54:59.229250+0000","last_unstale":"2024-09-13T14:57:02.864330+0000","last_undegraded":"2024-09-13T14:57:02.864330+0000","last_fullsized":"2024-09-13T14:57:02.864330+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:27:29.217218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":178,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.886198+0000","last_change":"2024-09-13T14:53:10.431597+0000","last_active":"2024-09-13T14:57:02.886198+0000","last_peered":"2024-09-13T14:57:02.886198+0000","last_clean":"2024-09-13T14:57:02.886198+0000","last_became_active":"2024-09-13T14:53:10.430897+0000","last_became_peered":"2024-09-13T14:53:10.430897+0000","last_unstale":"2024-09-13T14:57:02.886198+0000","last_undegraded":"2024-09-13T14:57:02.886198+0000","last_fullsized":"2024-09-13T14:57:02.886198+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:09:28.112054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":286,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.711260+0000","last_change":"2024-09-13T14:53:10.440740+0000","last_active":"2024-09-13T14:56:52.711260+0000","last_peered":"2024-09-13T14:56:52.711260+0000","last_clean":"2024-09-13T14:56:52.711260+0000","last_became_active":"2024-09-13T14:53:10.423498+0000","last_became_peered":"2024-09-13T14:53:10.423498+0000","last_unstale":"2024-09-13T14:56:52.711260+0000","last_undegraded":"2024-09-13T14:56:52.711260+0000","last_fullsized":"2024-09-13T14:56:52.711260+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:43:08.773406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":316,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.881739+0000","last_change":"2024-09-13T14:53:10.429291+0000","last_active":"2024-09-13T14:57:02.881739+0000","last_peered":"2024-09-13T14:57:02.881739+0000","last_clean":"2024-09-13T14:57:02.881739+0000","last_became_active":"2024-09-13T14:53:10.428533+0000","last_became_peered":"2024-09-13T14:53:10.428533+0000","last_unstale":"2024-09-13T14:57:02.881739+0000","last_undegraded":"2024-09-13T14:57:02.881739+0000","last_fullsized":"2024-09-13T14:57:02.881739+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:18:27.683915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":271,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.711308+0000","last_change":"2024-09-13T14:53:10.444612+0000","last_active":"2024-09-13T14:56:52.711308+0000","last_peered":"2024-09-13T14:56:52.711308+0000","last_clean":"2024-09-13T14:56:52.711308+0000","last_became_active":"2024-09-13T14:53:10.427868+0000","last_became_peered":"2024-09-13T14:53:10.427868+0000","last_unstale":"2024-09-13T14:56:52.711308+0000","last_undegraded":"2024-09-13T14:56:52.711308+0000","last_fullsized":"2024-09-13T14:56:52.711308+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:51:04.488560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":473,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.881379+0000","last_change":"2024-09-13T14:51:20.410963+0000","last_active":"2024-09-13T14:57:02.881379+0000","last_peered":"2024-09-13T14:57:02.881379+0000","last_clean":"2024-09-13T14:57:02.881379+0000","last_became_active":"2024-09-13T14:51:20.410333+0000","last_became_peered":"2024-09-13T14:51:20.410333+0000","last_unstale":"2024-09-13T14:57:02.881379+0000","last_undegraded":"2024-09-13T14:57:02.881379+0000","last_fullsized":"2024-09-13T14:57:02.881379+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:01:48.824154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":450,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.879299+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:57:02.879299+0000","last_peered":"2024-09-13T14:57:02.879299+0000","last_clean":"2024-09-13T14:57:02.879299+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:57:02.879299+0000","last_undegraded":"2024-09-13T14:57:02.879299+0000","last_fullsized":"2024-09-13T14:57:02.879299+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":74,"reported_epoch":301,"state":"active+clean","last_fresh":"2024-09-13T14:56:52.596954+0000","last_change":"2024-09-13T14:53:10.421063+0000","last_active":"2024-09-13T14:56:52.596954+0000","last_peered":"2024-09-13T14:56:52.596954+0000","last_clean":"2024-09-13T14:56:52.596954+0000","last_became_active":"2024-09-13T14:53:10.420718+0000","last_became_peered":"2024-09-13T14:53:10.420718+0000","last_unstale":"2024-09-13T14:56:52.596954+0000","last_undegraded":"2024-09-13T14:56:52.596954+0000","last_fullsized":"2024-09-13T14:56:52.596954+0000","mapping_epoch":270,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":271,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:07:13.952829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":47,"reported_epoch":302,"state":"active+clean","last_fresh":"2024-09-13T14:57:02.865696+0000","last_change":"2024-09-13T14:54:59.242750+0000","last_active":"2024-09-13T14:57:02.865696+0000","last_peered":"2024-09-13T14:57:02.865696+0000","last_clean":"2024-09-13T14:57:02.865696+0000","last_became_active":"2024-09-13T14:54:59.234348+0000","last_became_peered":"2024-09-13T14:54:59.234348+0000","last_unstale":"2024-09-13T14:57:02.865696+0000","last_undegraded":"2024-09-13T14:57:02.865696+0000","last_fullsized":"2024-09-13T14:57:02.865696+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:30:20.304575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":16,"stat_sum":{"num_bytes":2844655616,"num_objects":43406,"num_object_clones":0,"num_object_copies":130218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43406,"num_write_kb":2777984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8672444416,"data_stored":8672444416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43406,"ondisk_log_size":43406,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":5640,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":118784,"data_stored":21530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234446,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2449300,"kb_used_data":1421452,"kb_used_omap":1,"kb_used_meta":1027838,"kb_avail":91295852,"statfs":{"total":95995035648,"available":93486952448,"internally_reserved":0,"allocated":1455566848,"data_stored":1453910769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1052506571},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496084,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2397588,"kb_used_data":1437836,"kb_used_omap":1,"kb_used_meta":959742,"kb_avail":91347564,"statfs":{"total":95995035648,"available":93539905536,"internally_reserved":0,"allocated":1472344064,"data_stored":1470687982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":982776263},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757722,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1436936,"kb_used_data":359040,"kb_used_omap":1,"kb_used_meta":1077886,"kb_avail":92308216,"statfs":{"total":95995035648,"available":94523613184,"internally_reserved":0,"allocated":367656960,"data_stored":366009171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1103755718},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":0,"seq":128849019358,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650570,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2191236,"kb_used_data":1578108,"kb_used_omap":4,"kb_used_meta":613115,"kb_avail":91553916,"statfs":{"total":95995035648,"available":93751209984,"internally_reserved":0,"allocated":1615982592,"data_stored":1614338661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":627830319},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879505,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2309208,"kb_used_data":1818832,"kb_used_omap":6,"kb_used_meta":490361,"kb_avail":91435944,"statfs":{"total":95995035648,"available":93630406656,"internally_reserved":0,"allocated":1862483968,"data_stored":1860824929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":502130233},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435521,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1628284,"kb_used_data":810356,"kb_used_omap":5,"kb_used_meta":817914,"kb_avail":92116868,"statfs":{"total":95995035648,"available":94327672832,"internally_reserved":0,"allocated":829804544,"data_stored":828167666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":837544289},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926096,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1378056,"kb_used_data":935552,"kb_used_omap":5,"kb_used_meta":442490,"kb_avail":92367096,"statfs":{"total":95995035648,"available":94583906304,"internally_reserved":0,"allocated":958005248,"data_stored":956357281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":453109930},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,1,2,2,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":983236608,"data_stored":983236608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":854065152,"data_stored":854065152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1863122944,"data_stored":1863122944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1650524160,"data_stored":1650524160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":367329280,"data_stored":367329280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1469972480,"data_stored":1469972480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1484193792,"data_stored":1484193792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:57:25.310 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T14:57:25.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_7 pg_num 26 2024-09-13T14:57:25.387 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.425 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:25.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:25.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:25.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:25.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:25.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.707 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:25.707 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 57275 57259 111.805 72.5625 0.00650386 0.00893796 2024-09-13T14:57:25.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:25.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:25.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:25.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:25.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:25.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.083 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:26.124 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:26.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:26.226 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:25 smithi159 ceph-mon[102519]: pgmap v1094: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:26.251 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.283 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:25 smithi042 ceph-mon[108570]: pgmap v1094: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:26.283 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:25 smithi042 ceph-mon[113926]: pgmap v1094: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:26.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:57:26.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:26.377 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:26.479 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.506 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:26.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.708 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 58588 58572 110.903 82.0625 0.0106443 0.00900927 2024-09-13T14:57:26.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:26.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:26.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:26.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:26.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:26.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:27.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:27.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:27.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:27.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:27.294 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:27.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:27.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:27.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:27.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:27.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:27.571 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:27.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:27.708 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 59825 59809 109.915 77.3125 0.00547 0.00909273 2024-09-13T14:57:27.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:27.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:27.819 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:27.866 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:27.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:27.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:28.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.108 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:28.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:28.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:28.212 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:27 smithi042 ceph-mon[113926]: pgmap v1095: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:57:28.213 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:27 smithi042 ceph-mon[108570]: pgmap v1095: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:57:28.236 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:28.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:27 smithi159 ceph-mon[102519]: pgmap v1095: 99 pgs: 99 active+clean; 3.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T14:57:28.277 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:28.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:28.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:28.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:28.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:28.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:28.531 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:28.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:28.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:28.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:28.708 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 61005 60989 108.882 73.75 0.00797078 0.0091781 2024-09-13T14:57:28.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.809 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:28.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:28.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:28.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:28.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:28.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:29.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:29.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:29.205 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:29.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:29.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:29.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:29.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:29.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:29.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:29.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:29.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:29.708 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 62159 62143 107.861 72.125 0.012892 0.00926491 2024-09-13T14:57:29.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:29.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:29.902 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:29.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:29.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:29.973 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:57:29.979 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:29] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.51.0" 2024-09-13T14:57:30.030 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:30.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:30.172 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:29 smithi159 ceph-mon[102519]: pgmap v1096: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:57:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1369851729' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[108570]: pgmap v1096: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1369851729' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[113926]: pgmap v1096: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T14:57:30.341 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1369851729' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.342 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:57:30.342 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.342 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:30.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:30.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:30.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:30.571 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:30.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:30.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:30.689 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:30.689 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:30.710 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 63273 63257 106.821 69.625 0.0151337 0.00935488 2024-09-13T14:57:30.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:30.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:30.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:30.876 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:30.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:30.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:30.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.020 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:31.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:31.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: osdmap e305: 8 total, 8 up, 7 in 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: osdmap e305: 8 total, 8 up, 7 in 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:30 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:31.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:31.197 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: osdmap e305: 8 total, 8 up, 7 in 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:57:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:30 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:57:31.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:31.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:31.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:31.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:31.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:31.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:31.616 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:31.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 64540 64524 106.092 79.1875 0.0201504 0.00942042 2024-09-13T14:57:31.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:31.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:31.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:31.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:31.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:31.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:31.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:31.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:32.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:32.163 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:32.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:31 smithi042 ceph-mon[108570]: pgmap v1098: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:57:32.242 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:31 smithi042 ceph-mon[113926]: pgmap v1098: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:57:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:31 smithi159 ceph-mon[102519]: pgmap v1098: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-13T14:57:32.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:32.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:32.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:32.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:32.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:32.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:32.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:32.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:32.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:32.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:32.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:32.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:32.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 65784 65768 105.365 77.75 0.0107174 0.00948486 2024-09-13T14:57:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:57:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=cleanup t=2024-09-13T14:57:32.295847929Z level=info msg="Completed cleanup jobs" duration=2.856101ms 2024-09-13T14:57:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:57:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugins.update.checker t=2024-09-13T14:57:32.463657741Z level=info msg="Update check succeeded" duration=60.285292ms 2024-09-13T14:57:32.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:57:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana.update.checker t=2024-09-13T14:57:32.47711524Z level=info msg="Update check succeeded" duration=73.755529ms 2024-09-13T14:57:32.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:32.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:32.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:32.986 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:32.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.088 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:33.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:33.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:57:33.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:33.221 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/879421907' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:33.221 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/879421907' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:33.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:33.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.390 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:33.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/879421907' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:33.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:33.621 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:33.647 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:33.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:57:33.711127+0000 min lat: 0.00128182 max lat: 0.340148 avg lat: 0.00954714 2024-09-13T14:57:33.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:57:33.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 67028 67012 104.674 77.75 0.010097 0.00954714 2024-09-13T14:57:33.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:57:33.772 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:33.773 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:33.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:33.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:33.896 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:33.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:33.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:34.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:34.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:34.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:34.172 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:34.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:34.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:34 smithi042 ceph-mon[113926]: pgmap v1099: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-13T14:57:34.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:34 smithi042 ceph-mon[108570]: pgmap v1099: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-13T14:57:34.448 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:34.482 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:34.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:34.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:34 smithi159 ceph-mon[102519]: pgmap v1099: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2024-09-13T14:57:34.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:34.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:34.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 68295 68279 104.053 79.1875 0.00949539 0.00960529 2024-09-13T14:57:34.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:34.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:34.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:34.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:34.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:34.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:34.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:35.008 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:35.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:35.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:35.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:35.236 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:35.263 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:35.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.402 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:35.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:35.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:35.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:35.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:35.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:35.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 69536 69520 103.422 77.5625 0.0114082 0.00966349 2024-09-13T14:57:35.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:35.797 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:35.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:35.938 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:35.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:35.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:36.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:36.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:36.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:36.212 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:36.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:36 smithi042 ceph-mon[108570]: pgmap v1100: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:57:36.275 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:36 smithi042 ceph-mon[113926]: pgmap v1100: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:57:36.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:36.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:36.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.485 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:36 smithi159 ceph-mon[102519]: pgmap v1100: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-13T14:57:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:36.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:36.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:36.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:36.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 70710 70694 102.723 73.375 0.0175753 0.00972902 2024-09-13T14:57:36.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:36.739 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:57:36.739 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:57:36.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:36.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:36.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:36.900 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:36.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:36.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:37.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:57:37.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:37.198 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:37.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:37.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.347 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:37.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:37.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:37.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:37.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T14:57:37.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:37.711 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 71841 71825 101.995 70.6875 0.00329018 0.00979508 2024-09-13T14:57:37.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:37.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:37.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:37.923 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:37.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:38.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:38.048 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:38.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:38.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:38.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:38.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:38.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:38 smithi042 ceph-mon[108570]: pgmap v1101: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:57:38.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:38 smithi042 ceph-mon[113926]: pgmap v1101: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:57:38.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:38.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:38.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:38 smithi159 ceph-mon[102519]: pgmap v1101: 99 pgs: 99 active+clean; 4.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T14:57:38.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:38.589 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:38.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:38.693 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:38.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 72935 72919 101.245 68.375 0.0134804 0.00987117 2024-09-13T14:57:38.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:38.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:38.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:38.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:38.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:38.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:39.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:39.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:39.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:39.163 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:39.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:39.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:39.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:39.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:39.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:39.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:39.527 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:39.555 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:39.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:39.657 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:39.698 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:39.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 74000 73984 100.492 66.5625 0.00902193 0.00994417 2024-09-13T14:57:39.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:39.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:39.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:39.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:39.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:39.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:40.042 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:39] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:57:40.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:40.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:40.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:40.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:40.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[108570]: pgmap v1102: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:57:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1590601821' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[113926]: pgmap v1102: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:57:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1590601821' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:40.407 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 12 pg_num to 26 2024-09-13T14:57:40.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:40.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:40 smithi159 ceph-mon[102519]: pgmap v1102: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-09-13T14:57:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1590601821' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T14:57:40.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:40.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:40.617 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:40.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 75145 75129 99.8761 71.5625 0.0038332 0.0100056 2024-09-13T14:57:40.743 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:40.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:40.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:40.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:40.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:40.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:40.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:41.019 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:41.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:41.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:41.171 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:41.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:41.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:41.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:41.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:57:41.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[108570]: osdmap e306: 8 total, 8 up, 7 in 2024-09-13T14:57:41.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:57:41.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:57:41.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[113926]: osdmap e306: 8 total, 8 up, 7 in 2024-09-13T14:57:41.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:41 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:57:41.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:41.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:41 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num", "val": "26"}]': finished 2024-09-13T14:57:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:41 smithi159 ceph-mon[102519]: osdmap e306: 8 total, 8 up, 7 in 2024-09-13T14:57:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:41 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T14:57:41.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.576 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:41.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:41.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:41.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 75509 75493 98.2695 22.75 0.0488832 0.0100402 2024-09-13T14:57:41.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:41.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:41.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:41.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:41.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:41.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:42.187 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.224 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:42.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[113926]: pgmap v1104: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:57:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:57:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[113926]: osdmap e307: 8 total, 8 up, 7 in 2024-09-13T14:57:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[108570]: pgmap v1104: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:57:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:57:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:42 smithi042 ceph-mon[108570]: osdmap e307: 8 total, 8 up, 7 in 2024-09-13T14:57:42.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:42.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:42.480 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:42.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:42 smithi159 ceph-mon[102519]: pgmap v1104: 99 pgs: 99 active+clean; 4.4 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T14:57:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:42 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T14:57:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:42 smithi159 ceph-mon[102519]: osdmap e307: 8 total, 8 up, 7 in 2024-09-13T14:57:42.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:42.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:42.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 76286 76270 97.2552 48.5625 0.0131099 0.0102768 2024-09-13T14:57:42.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:42.764 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:42.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:42.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:42.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:42.902 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:42.989 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:43.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.147 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.249 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:43 smithi042 ceph-mon[113926]: osdmap e308: 8 total, 8 up, 7 in 2024-09-13T14:57:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:43 smithi042 ceph-mon[108570]: osdmap e308: 8 total, 8 up, 7 in 2024-09-13T14:57:43.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:43.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:43 smithi159 ceph-mon[102519]: osdmap e308: 8 total, 8 up, 7 in 2024-09-13T14:57:43.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:43.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:43.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:43.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:43.698 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:43.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 77736 77720 97.1224 90.625 0.0133724 0.0102904 2024-09-13T14:57:43.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:43.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:43.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:43.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:43.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:43.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:43.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:44.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:44.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:44.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.228 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.326 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[108570]: pgmap v1107: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:44.326 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[108570]: 12.9 scrub starts 2024-09-13T14:57:44.326 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[108570]: 12.3 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[108570]: 12.2 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[108570]: 12.7 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[113926]: pgmap v1107: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[113926]: 12.9 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[113926]: 12.3 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[113926]: 12.2 scrub starts 2024-09-13T14:57:44.327 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:44 smithi042 ceph-mon[113926]: 12.7 scrub starts 2024-09-13T14:57:44.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:44.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:44 smithi159 ceph-mon[102519]: pgmap v1107: 109 pgs: 7 peering, 10 unknown, 92 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T14:57:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:44 smithi159 ceph-mon[102519]: 12.9 scrub starts 2024-09-13T14:57:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:44 smithi159 ceph-mon[102519]: 12.3 scrub starts 2024-09-13T14:57:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:44 smithi159 ceph-mon[102519]: 12.2 scrub starts 2024-09-13T14:57:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:44 smithi159 ceph-mon[102519]: 12.7 scrub starts 2024-09-13T14:57:44.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:44.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:44.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:44.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 79176 79160 96.9826 90 0.0200806 0.0103056 2024-09-13T14:57:44.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:44.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:44.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:44.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:44.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:44.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:45.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:45.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:45.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:45.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:45.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[108570]: 12.19 scrub starts 2024-09-13T14:57:45.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[108570]: 12.13 scrub starts 2024-09-13T14:57:45.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[108570]: 12.8 scrub starts 2024-09-13T14:57:45.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[108570]: 12.17 scrub starts 2024-09-13T14:57:45.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[113926]: 12.19 scrub starts 2024-09-13T14:57:45.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[113926]: 12.13 scrub starts 2024-09-13T14:57:45.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[113926]: 12.8 scrub starts 2024-09-13T14:57:45.255 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:45 smithi042 ceph-mon[113926]: 12.17 scrub starts 2024-09-13T14:57:45.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:45.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:45.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:45.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:45 smithi159 ceph-mon[102519]: 12.19 scrub starts 2024-09-13T14:57:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:45 smithi159 ceph-mon[102519]: 12.13 scrub starts 2024-09-13T14:57:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:45 smithi159 ceph-mon[102519]: 12.8 scrub starts 2024-09-13T14:57:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:45 smithi159 ceph-mon[102519]: 12.17 scrub starts 2024-09-13T14:57:45.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:45.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:45.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:45.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 15 80616 80601 96.8494 90.0625 0.00809599 0.0103207 2024-09-13T14:57:45.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:45.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:45.769 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:45.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:45.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:45.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:45.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:45.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:45.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:46.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.201 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[108570]: pgmap v1108: 109 pgs: 7 peering, 9 unknown, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:57:46.201 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[108570]: 12.12 scrub starts 2024-09-13T14:57:46.201 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[108570]: 12.6 scrub starts 2024-09-13T14:57:46.201 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[108570]: 12.16 scrub starts 2024-09-13T14:57:46.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[113926]: pgmap v1108: 109 pgs: 7 peering, 9 unknown, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:57:46.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[113926]: 12.12 scrub starts 2024-09-13T14:57:46.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[113926]: 12.6 scrub starts 2024-09-13T14:57:46.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:46 smithi042 ceph-mon[113926]: 12.16 scrub starts 2024-09-13T14:57:46.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:46.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.245 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:46.346 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:46.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:46.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:46 smithi159 ceph-mon[102519]: pgmap v1108: 109 pgs: 7 peering, 9 unknown, 93 active+clean; 4.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T14:57:46.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:46 smithi159 ceph-mon[102519]: 12.12 scrub starts 2024-09-13T14:57:46.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:46 smithi159 ceph-mon[102519]: 12.6 scrub starts 2024-09-13T14:57:46.400 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:46 smithi159 ceph-mon[102519]: 12.16 scrub starts 2024-09-13T14:57:46.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:46.501 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:46.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:46.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:46.714 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 82047 82031 96.7083 89.375 0.00736024 0.0103351 2024-09-13T14:57:46.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:46.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:46.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:46.901 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:46.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:46.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:47.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:47.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:47.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:47.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:47.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:47.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:47.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.304 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:47.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:47.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:47 smithi042 ceph-mon[113926]: 12.18 scrub starts 2024-09-13T14:57:47.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:47 smithi042 ceph-mon[108570]: 12.18 scrub starts 2024-09-13T14:57:47.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:47.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:47.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:47.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:47 smithi159 ceph-mon[102519]: 12.18 scrub starts 2024-09-13T14:57:47.531 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:47.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:47.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:47.707 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:47.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:47.713 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 83382 83366 96.4624 83.4375 0.00564917 0.0103611 2024-09-13T14:57:47.809 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:47.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:47.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:47.984 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:47.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:47.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:48.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T14:57:48.100 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:57:48.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T14:57:48.232 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.270 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[113926]: pgmap v1109: 109 pgs: 7 peering, 7 unknown, 95 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:57:48.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/31638402' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[113926]: 12.9 scrub starts 2024-09-13T14:57:48.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[108570]: pgmap v1109: 109 pgs: 7 peering, 7 unknown, 95 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:57:48.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/31638402' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:48 smithi042 ceph-mon[108570]: 12.9 scrub starts 2024-09-13T14:57:48.372 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:48.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.413 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:48.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:48 smithi159 ceph-mon[102519]: pgmap v1109: 109 pgs: 7 peering, 7 unknown, 95 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 758 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T14:57:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/31638402' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:48 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:57:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:48 smithi159 ceph-mon[102519]: 12.9 scrub starts 2024-09-13T14:57:48.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:48.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:48.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:48.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:48.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:48.716 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 84762 84746 96.2727 86.25 0.0115417 0.0103813 2024-09-13T14:57:48.737 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:57:48.737 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:57:48.738 INFO:tasks.thrashosds.thrasher:Killing osd 0, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:57:48.738 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T14:57:48.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0 2024-09-13T14:57:48.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:48.824 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:48.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:48.926 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:48.950 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:57:48 smithi042 systemd[1]: Stopping Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T14:57:48.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:48.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:48.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:49.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:49.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: osdmap e309: 8 total, 8 up, 7 in 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: 12.19 scrub starts 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[113926]: 12.7 scrub starts 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: osdmap e309: 8 total, 8 up, 7 in 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: 12.19 scrub starts 2024-09-13T14:57:49.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-mon[108570]: 12.7 scrub starts 2024-09-13T14:57:49.337 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.395 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.497 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: osdmap e309: 8 total, 8 up, 7 in 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: 12.19 scrub starts 2024-09-13T14:57:49.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:49 smithi159 ceph-mon[102519]: 12.7 scrub starts 2024-09-13T14:57:49.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.593 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T14:57:49.594 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T14:57:49.629 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:57:49.630 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:57:49.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:49.663 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:57:49.664 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:57:49.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:49.697 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:57:49.697 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:57:49.716 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 85583 85567 95.4699 51.3125 0.00336178 0.0104253 2024-09-13T14:57:49.727 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:57:49.727 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:57:49.760 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:57:49.760 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:57:49.776 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:49.799 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:57:49.800 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:57:49.813 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:49.837 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:57:49.837 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:57:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:49] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-13T14:57:49.916 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:49.935 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:57:49.935 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:57:49.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124021 10.14 10:2ba3697e:::benchmark_data_smithi042_191918_object428:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.47913671100002, 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1046680210000002, 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.936 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124021 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452802+0000", 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452804+0000", 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7010000000000001e-06 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452810+0000", 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5620000000000003e-06 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452828+0000", 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8241999999999999e-05 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.452836+0000", 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1459999999999996e-06 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453443+0000", 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060669599999999999 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.453453+0000", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0169999999999999e-05 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.376987+0000", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.92353413299999998 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377000+0000", 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3369e-05 2024-09-13T14:57:49.940 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494162+0000", 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.117161158 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.494306+0000", 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014396600000000001 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.548321+0000", 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.054015170000000001 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555032+0000", 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006711308 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555051+0000", 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9389e-05 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555093+0000", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1325000000000002e-05 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.555161+0000", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8522999999999994e-05 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557470+0000", 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023091629999999999 2024-09-13T14:57:49.943 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124045 10.14 10:2fb89687:::benchmark_data_smithi042_191918_object452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.46029808799994, 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0906675360000002, 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.944 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124045 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471641+0000", 2024-09-13T14:57:49.945 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471642+0000", 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3349999999999999e-06 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471647+0000", 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.955e-06 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.946 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471659+0000", 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2055000000000001e-05 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471666+0000", 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0890000000000002e-06 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471724+0000", 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7751000000000003e-05 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.471734+0000", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3589999999999997e-06 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377030+0000", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90529651300000002 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377038+0000", 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8059999999999995e-06 2024-09-13T14:57:49.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552572+0000", 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1755340999999999 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.552679+0000", 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010687 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.553038+0000", 2024-09-13T14:57:49.950 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000359148 2024-09-13T14:57:49.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:49.951 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.951 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.559991+0000", 2024-09-13T14:57:49.951 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069528999999999997 2024-09-13T14:57:49.951 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.951 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560059+0000", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7787999999999995e-05 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560076+0000", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7073999999999999e-05 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.560136+0000", 2024-09-13T14:57:49.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0300999999999997e-05 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562309+0000", 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021724919999999998 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124123 10.14 10:2f92ce95:::benchmark_data_smithi042_191918_object530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.953 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.39306022599999, 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0288850589999998, 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124123 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.954 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538879+0000", 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538880+0000", 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4339999999999999e-06 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538886+0000", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.203e-06 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538900+0000", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4361e-05 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538906+0000", 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.19e-06 2024-09-13T14:57:49.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538972+0000", 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5877000000000006e-05 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.538985+0000", 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2565000000000001e-05 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377062+0000", 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.83807765400000001 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377069+0000", 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8399999999999997e-06 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557527+0000", 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.180457504 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.557691+0000", 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164358 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.558112+0000", 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000421579 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565029+0000", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.006916663 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565048+0000", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9318e-05 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565106+0000", 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7472999999999997e-05 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.565161+0000", 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4979000000000003e-05 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567764+0000", 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026030609999999998 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124160 10.14 10:2a1875cd:::benchmark_data_smithi042_191918_object567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.36306007099995, 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0036828569999998, 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.962 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124160 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568879+0000", 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568881+0000", 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.649e-06 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568886+0000", 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1680000000000003e-06 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.964 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568898+0000", 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2198000000000001e-05 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568905+0000", 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1040000000000001e-06 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568973+0000", 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7587000000000002e-05 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.965 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.568983+0000", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0091e-05 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377092+0000", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.80810927200000005 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377099+0000", 2024-09-13T14:57:49.966 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.754e-06 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562371+0000", 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.185272007 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562528+0000", 2024-09-13T14:57:49.967 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015748699999999999 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.562968+0000", 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00043982 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.968 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570098+0000", 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0071301250000000002 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570121+0000", 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2694e-05 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570194+0000", 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2587000000000001e-05 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.969 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.570259+0000", 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5026999999999999e-05 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572562+0000", 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023032870000000002 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.970 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124175 10.14 10:2bb4c019:::benchmark_data_smithi042_191918_object582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.34982448000005, 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.996287562, 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124175 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.971 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582115+0000", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582116+0000", 2024-09-13T14:57:49.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.697e-06 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582122+0000", 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.31e-06 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.582135+0000", 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-05 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.625825+0000", 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.04369029 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626947+0000", 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011223139999999999 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.626958+0000", 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.039e-05 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377122+0000", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.75016387699999998 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377128+0000", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.708e-06 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.567848+0000", 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.19071927 2024-09-13T14:57:49.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568046+0000", 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000198648 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.568459+0000", 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041277499999999999 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.976 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.575988+0000", 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075292850000000001 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576003+0000", 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4636e-05 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576043+0000", 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0141000000000001e-05 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.576115+0000", 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2324999999999996e-05 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578402+0000", 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022867859999999999 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124203 10.14 10:2df4aadf:::benchmark_data_smithi042_191918_object610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.978 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.30020655500005, 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9513996520000001, 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124203 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.979 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631733+0000", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631734+0000", 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.393e-06 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631739+0000", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8720000000000001e-06 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631751+0000", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2089000000000001e-05 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631758+0000", 2024-09-13T14:57:49.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5310000000000004e-06 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631820+0000", 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1235999999999994e-05 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.631830+0000", 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0087e-05 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.982 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377151+0000", 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.74532109400000002 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377157+0000", 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.567e-06 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572621+0000", 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1954635090000001 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.572838+0000", 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000217176 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.984 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.573258+0000", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041985200000000002 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578468+0000", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052102149999999998 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578487+0000", 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9037000000000001e-05 2024-09-13T14:57:49.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581071+0000", 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025835509999999999 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.581132+0000", 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0745000000000002e-05 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583132+0000", 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002000698 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124303 10.14 10:2cafaf1c:::benchmark_data_smithi042_191918_object710:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.20935252300001, 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8779041919999999, 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.987 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124303 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722587+0000", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722588+0000", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.159e-06 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722592+0000", 2024-09-13T14:57:49.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1330000000000001e-06 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722602+0000", 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0475000000000001e-05 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722609+0000", 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6880000000000002e-06 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722665+0000", 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5510000000000002e-05 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.722675+0000", 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0122000000000001e-05 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377180+0000", 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.65450549800000002 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377187+0000", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7050000000000002e-06 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578628+0000", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2014406769999999 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.578774+0000", 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014649800000000001 2024-09-13T14:57:49.992 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.579165+0000", 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039142000000000002 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.585995+0000", 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068292939999999996 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:49.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590672+0000", 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046776760000000004 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597892+0000", 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072192139999999998 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597941+0000", 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9373000000000003e-05 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600491+0000", 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025497499999999999 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124330 10.14 10:2826c96e:::benchmark_data_smithi042_191918_object737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.19001325800002, 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.867454178, 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:49.995 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:49.996 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:49.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:49.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124330 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.997 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741926+0000", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741928+0000", 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9069999999999999e-06 2024-09-13T14:57:49.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741934+0000", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5509999999999996e-06 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741957+0000", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2571000000000001e-05 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.741966+0000", 2024-09-13T14:57:49.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6219999999999998e-06 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742027+0000", 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1544000000000006e-05 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.742037+0000", 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0363999999999999e-05 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.000 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377210+0000", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.63517246999999999 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377217+0000", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9410000000000001e-06 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583202+0000", 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2059855310000001 2024-09-13T14:57:50.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583348+0000", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000145131 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.583747+0000", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039933999999999998 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590701+0000", 2024-09-13T14:57:50.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069539379999999998 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590715+0000", 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4312e-05 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606747+0000", 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016031553 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.606846+0000", 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9411000000000001e-05 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609380+0000", 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025339920000000001 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124347 10.14 10:2b51d2b1:::benchmark_data_smithi042_191918_object754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:50.004 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.17252109699996, 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.852510501, 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124347 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.005 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759418+0000", 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759419+0000", 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6000000000000004e-07 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759421+0000", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7700000000000002e-06 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759428+0000", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1199999999999996e-06 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759435+0000", 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.336e-06 2024-09-13T14:57:50.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759494+0000", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9657000000000002e-05 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.759505+0000", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0743e-05 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377240+0000", 2024-09-13T14:57:50.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.61773501500000005 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377247+0000", 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5420000000000002e-06 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586075+0000", 2024-09-13T14:57:50.009 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.208828676 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586227+0000", 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015113099999999999 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.586620+0000", 2024-09-13T14:57:50.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039343899999999998 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590721+0000", 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041013100000000004 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590746+0000", 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5072e-05 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609450+0000", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018703280999999999 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.609614+0000", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164084 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.611929+0000", 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023147649999999999 2024-09-13T14:57:50.012 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124381 10.14 10:2889f54c:::benchmark_data_smithi042_191918_object788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.141882442, 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.831978273, 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.013 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124381 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790057+0000", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1799999999999995e-07 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790060+0000", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.774e-06 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790069+0000", 2024-09-13T14:57:50.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7450000000000001e-06 2024-09-13T14:57:50.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790074+0000", 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5140000000000001e-06 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790150+0000", 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5729000000000006e-05 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.790163+0000", 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2855e-05 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377270+0000", 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.58710728099999998 2024-09-13T14:57:50.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377276+0000", 2024-09-13T14:57:50.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3759999999999996e-06 2024-09-13T14:57:50.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588559+0000", 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2112829700000001 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.588744+0000", 2024-09-13T14:57:50.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018437400000000001 2024-09-13T14:57:50.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:50.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.589096+0000", 2024-09-13T14:57:50.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035260100000000002 2024-09-13T14:57:50.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590759+0000", 2024-09-13T14:57:50.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016620910000000001 2024-09-13T14:57:50.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600557+0000", 2024-09-13T14:57:50.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097980979999999999 2024-09-13T14:57:50.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619838+0000", 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019281598 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.619895+0000", 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6721000000000001e-05 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622035+0000", 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002140028 2024-09-13T14:57:50.032 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.033 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.033 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.033 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124511 10.14 10:2e9e45ad:::benchmark_data_smithi042_191918_object918:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 567.00523189099999, 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6976293679999999, 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.034 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124511 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926707+0000", 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926709+0000", 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5600000000000001e-06 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926715+0000", 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0560000000000001e-06 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926732+0000", 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6949e-05 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926740+0000", 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3559999999999993e-06 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926811+0000", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0926000000000005e-05 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.926822+0000", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0509999999999999e-05 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377300+0000", 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.45047793699999999 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377306+0000", 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6189999999999996e-06 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590780+0000", 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.213474116 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.590981+0000", 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000200495 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.591320+0000", 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00033930899999999999 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.597842+0000", 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0065215780000000001 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600611+0000", 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027691840000000001 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622109+0000", 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021498397999999998 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.622172+0000", 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3184999999999999e-05 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624337+0000", 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021641899999999999 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:124519 10.14 10:2894197e:::benchmark_data_smithi042_191918_object926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e213)", 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 566.996151865, 2024-09-13T14:57:50.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6923704129999999, 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:57:50.045 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 124519 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935787+0000", 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935789+0000", 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5149999999999999e-06 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935794+0000", 2024-09-13T14:57:50.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8069999999999999e-06 2024-09-13T14:57:50.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935806+0000", 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721999999999999e-05 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935813+0000", 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8079999999999999e-06 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935890+0000", 2024-09-13T14:57:50.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7139999999999997e-05 2024-09-13T14:57:50.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:22.935901+0000", 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1178e-05 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377329+0000", 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.44142803000000003 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:57:50.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:23.377336+0000", 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5459999999999997e-06 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593242+0000", 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.215905585 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593400+0000", 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015875199999999999 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,4", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.593797+0000", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039664300000000002 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600508+0000", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067110399999999997 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.600684+0000", 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017580100000000001 2024-09-13T14:57:50.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624404+0000", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023720393999999999 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.624466+0000", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2217000000000004e-05 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:48:24.628158+0000", 2024-09-13T14:57:50.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036912350000000002 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:59618 12.6 12:627f8d89:::benchmark_data_smithi042_260592_object59617:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:27.528322+0000", 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 22.403617574999998, 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.035778270000000001, 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.055 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 59618 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528322+0000", 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528322+0000", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528324+0000", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0509999999999999e-06 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528362+0000", 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8699000000000003e-05 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528428+0000", 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5287e-05 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528456+0000", 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8008e-05 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528615+0000", 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015892599999999999 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528806+0000", 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000191542 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 0,7", 2024-09-13T14:57:50.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.528994+0000", 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018778100000000001 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.531349+0000", 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023549930000000001 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.532343+0000", 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00099392300000000007 2024-09-13T14:57:50.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.563816+0000", 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.031473056999999999 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.564045+0000", 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000229342 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.564100+0000", 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4660999999999997e-05 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:59632 12.d 12:b7d6995c:::benchmark_data_smithi042_260592_object59631:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-13T14:57:50.063 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:27.544955+0000", 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 22.386983729000001, 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.034228007999999997, 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 59632 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.544955+0000", 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.544955+0000", 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.544957+0000", 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9769999999999999e-06 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545014+0000", 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6709999999999997e-05 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545033+0000", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9211e-05 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545041+0000", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0579999999999998e-06 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545111+0000", 2024-09-13T14:57:50.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9740000000000007e-05 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545250+0000", 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013935099999999999 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.545390+0000", 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000139844 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.548369+0000", 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002978643 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.550620+0000", 2024-09-13T14:57:50.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022507009999999999 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.578852+0000", 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.028232770000000001 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.579027+0000", 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017416799999999999 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:27.579183+0000", 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000156835 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.071 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:60872 12.d 12:b98b63d5:::benchmark_data_smithi042_260592_object60871:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.592046+0000", 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.339892913, 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033110647999999999, 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60872 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592046+0000", 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592046+0000", 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592048+0000", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6390000000000001e-06 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592072+0000", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3961999999999999e-05 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592084+0000", 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2578999999999999e-05 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.592092+0000", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1810000000000004e-06 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602781+0000", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010689839 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602927+0000", 2024-09-13T14:57:50.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014555899999999999 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.603059+0000", 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013181099999999999 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.607918+0000", 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048587609999999996 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.611607+0000", 2024-09-13T14:57:50.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036892660000000001 2024-09-13T14:57:50.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.625067+0000", 2024-09-13T14:57:50.078 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013460445 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.625101+0000", 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3776000000000003e-05 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.625157+0000", 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5829999999999999e-05 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:60879 12.5 e304/298 12:a61517e6:::benchmark_data_smithi042_260592_object60878:head v 304'3843, mlcod=304'3836)", 2024-09-13T14:57:50.080 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.598790+0000", 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.333149527, 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033291111999999998, 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.598790+0000", 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.081 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.598790+0000", 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.598790+0000", 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0400000000000005e-07 2024-09-13T14:57:50.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.598898+0000", 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107496 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.601759+0000", 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028614510000000001 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.601767+0000", 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8630000000000008e-06 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.620880+0000", 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019113023 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.620959+0000", 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8903e-05 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.631944+0000", 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010985152999999999 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.632081+0000", 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013641900000000001 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.085 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:60885 12.5 e304/298 12:a300ca73:::benchmark_data_smithi042_260592_object60884:head v 304'3845, mlcod=304'3836)", 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.602617+0000", 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.329321849999999, 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033005531999999997, 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.086 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602617+0000", 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602617+0000", 2024-09-13T14:57:50.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602622+0000", 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4279999999999998e-06 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602723+0000", 2024-09-13T14:57:50.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101141 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602730+0000", 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3710000000000002e-06 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.602736+0000", 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7699999999999998e-06 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.625154+0000", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022417986000000001 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.625176+0000", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2121e-05 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.634615+0000", 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0094389009999999995 2024-09-13T14:57:50.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.635623+0000", 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010078140000000001 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:60892 12.5 e304/298 12:a44ccb32:::benchmark_data_smithi042_260592_object60891:head v 304'3847, mlcod=304'3837)", 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.606931+0000", 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.325007801000002, 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.032183340999999997, 2024-09-13T14:57:50.091 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.606931+0000", 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.606931+0000", 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.606932+0000", 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9299999999999997e-07 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.607048+0000", 2024-09-13T14:57:50.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011539 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.607057+0000", 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3209999999999994e-06 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.607063+0000", 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6290000000000004e-06 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.630040+0000", 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022976980000000001 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.630059+0000", 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8298999999999999e-05 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.639065+0000", 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0090064370000000008 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.639115+0000", 2024-09-13T14:57:50.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9492e-05 2024-09-13T14:57:50.097 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.097 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.097 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.098 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:60959 12.7 12:eddca3d1:::benchmark_data_smithi042_260592_object60958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-13T14:57:50.098 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.663129+0000", 2024-09-13T14:57:50.098 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.268809848, 2024-09-13T14:57:50.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.032365215000000003, 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60959 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663129+0000", 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663129+0000", 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663131+0000", 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3319999999999999e-06 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663155+0000", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4842e-05 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663173+0000", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7186000000000001e-05 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.663181+0000", 2024-09-13T14:57:50.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2479999999999996e-06 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.668579+0000", 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0053981319999999999 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.668701+0000", 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000121589 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.668827+0000", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012679 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.671822+0000", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029947400000000001 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.687141+0000", 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015318500000000001 2024-09-13T14:57:50.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.695338+0000", 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081974609999999996 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.695440+0000", 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010184899999999999 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.695494+0000", 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4546000000000003e-05 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:61187 12.d 12:bbd6a72e:::benchmark_data_smithi042_260592_object61186:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e304)", 2024-09-13T14:57:50.105 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:28.861372+0000", 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 21.070566916000001, 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.035105059000000001, 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 61187 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:57:50.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.861372+0000", 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.861372+0000", 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.861373+0000", 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2600000000000001e-07 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.862415+0000", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001041534 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.862432+0000", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7883000000000001e-05 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.862442+0000", 2024-09-13T14:57:50.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3160000000000003e-06 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.870094+0000", 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076524760000000001 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.870226+0000", 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000131347 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.870385+0000", 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000159128 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:57:50.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.874754+0000", 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043694019999999997 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.874789+0000", 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4477999999999998e-05 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.895000+0000", 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020211223 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.896421+0000", 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014208339999999999 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:28.896477+0000", 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6611999999999999e-05 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:57:50.112 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:57:50.113 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:57:50.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.154 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.256 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:50.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:50.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:50.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:50.402 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:50.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: pgmap v1111: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.3 scrub starts 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.2 scrub starts 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: osdmap e310: 8 total, 8 up, 7 in 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3457431880' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.19 scrub ok 2024-09-13T14:57:50.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.9 scrub starts 2024-09-13T14:57:50.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.7 scrub ok 2024-09-13T14:57:50.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:50 smithi159 ceph-mon[102519]: 12.6 scrub starts 2024-09-13T14:57:50.504 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:50.526 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: pgmap v1111: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.3 scrub starts 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.2 scrub starts 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: osdmap e310: 8 total, 8 up, 7 in 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3457431880' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:50.580 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.19 scrub ok 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.9 scrub starts 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.7 scrub ok 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[108570]: 12.6 scrub starts 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: pgmap v1111: 109 pgs: 109 active+clean; 4.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.3 scrub starts 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.2 scrub starts 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: osdmap e310: 8 total, 8 up, 7 in 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3457431880' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.19 scrub ok 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.9 scrub starts 2024-09-13T14:57:50.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.7 scrub ok 2024-09-13T14:57:50.582 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:50 smithi042 ceph-mon[113926]: 12.6 scrub starts 2024-09-13T14:57:50.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:50.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:50.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:50.716 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 85710 85694 93.9345 7.9375 0.00286101 0.0104214 2024-09-13T14:57:50.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:50.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:50.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:50.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:50.902 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:50.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:50.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:51.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.028 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:51.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:51.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:51.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:51.255 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:51.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:51.315 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:57:51.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:51.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:51.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.2 scrub ok 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.3 scrub ok 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.13 scrub starts 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.18 scrub starts 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: osdmap e311: 8 total, 8 up, 7 in 2024-09-13T14:57:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.18 scrub ok 2024-09-13T14:57:51.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.9 scrub ok 2024-09-13T14:57:51.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.6 scrub ok 2024-09-13T14:57:51.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:57:51.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:51.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:51 smithi159 ceph-mon[102519]: 12.17 scrub starts 2024-09-13T14:57:51.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:51.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:51.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.2 scrub ok 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.3 scrub ok 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.13 scrub starts 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.18 scrub starts 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: osdmap e311: 8 total, 8 up, 7 in 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.18 scrub ok 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.9 scrub ok 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.6 scrub ok 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[108570]: 12.17 scrub starts 2024-09-13T14:57:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.2 scrub ok 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.3 scrub ok 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.13 scrub starts 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.18 scrub starts 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: osdmap e311: 8 total, 8 up, 7 in 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.18 scrub ok 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.9 scrub ok 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.6 scrub ok 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:57:51.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:51 smithi042 ceph-mon[113926]: 12.17 scrub starts 2024-09-13T14:57:51.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:51.661 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:51.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:51.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 86075 86059 92.7072 22.8125 0.0225763 0.0107658 2024-09-13T14:57:51.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:51.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:51.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:51.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T14:57:51.909 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:51.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:51.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:52.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:52.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:52.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: 12.13 scrub ok 2024-09-13T14:57:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: pgmap v1114: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:57:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: 12.8 scrub starts 2024-09-13T14:57:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: 12.8 scrub ok 2024-09-13T14:57:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:57:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: osdmap e312: 8 total, 8 up, 7 in 2024-09-13T14:57:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: 12.17 scrub ok 2024-09-13T14:57:52.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[108570]: 12.16 scrub starts 2024-09-13T14:57:52.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: 12.13 scrub ok 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: pgmap v1114: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: 12.8 scrub starts 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: 12.8 scrub ok 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: osdmap e312: 8 total, 8 up, 7 in 2024-09-13T14:57:52.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: 12.17 scrub ok 2024-09-13T14:57:52.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:52 smithi042 ceph-mon[113926]: 12.16 scrub starts 2024-09-13T14:57:52.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:52.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: 12.13 scrub ok 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: pgmap v1114: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: 12.8 scrub starts 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: 12.8 scrub ok 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: osdmap e312: 8 total, 8 up, 7 in 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: 12.17 scrub ok 2024-09-13T14:57:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:52 smithi159 ceph-mon[102519]: 12.16 scrub starts 2024-09-13T14:57:52.545 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:52.568 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:52.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:52.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:52.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:52.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 86339 86323 91.4158 16.5 0.00226147 0.0107522 2024-09-13T14:57:52.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:52.918 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:52.939 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:52.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:52.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:53.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:53.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:53.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:53.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:53.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:53.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:53.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:53.417 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:53.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:53 smithi159 ceph-mon[102519]: 12.12 scrub starts 2024-09-13T14:57:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:53 smithi159 ceph-mon[102519]: osdmap e313: 8 total, 8 up, 7 in 2024-09-13T14:57:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:53 smithi159 ceph-mon[102519]: 12.12 scrub ok 2024-09-13T14:57:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:53 smithi159 ceph-mon[102519]: 12.16 scrub ok 2024-09-13T14:57:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:53 smithi159 ceph-mon[102519]: osdmap e314: 8 total, 8 up, 7 in 2024-09-13T14:57:53.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:53.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[108570]: 12.12 scrub starts 2024-09-13T14:57:53.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[108570]: osdmap e313: 8 total, 8 up, 7 in 2024-09-13T14:57:53.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[108570]: 12.12 scrub ok 2024-09-13T14:57:53.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[108570]: 12.16 scrub ok 2024-09-13T14:57:53.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[108570]: osdmap e314: 8 total, 8 up, 7 in 2024-09-13T14:57:53.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[113926]: 12.12 scrub starts 2024-09-13T14:57:53.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[113926]: osdmap e313: 8 total, 8 up, 7 in 2024-09-13T14:57:53.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[113926]: 12.12 scrub ok 2024-09-13T14:57:53.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[113926]: 12.16 scrub ok 2024-09-13T14:57:53.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:53 smithi042 ceph-mon[113926]: osdmap e314: 8 total, 8 up, 7 in 2024-09-13T14:57:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.604 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:53.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:53.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:53.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:57:53.717156+0000 min lat: 0.00128182 max lat: 2.19784 avg lat: 0.0110808 2024-09-13T14:57:53.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:57:53.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 86626 86610 90.1914 17.9375 0.0232248 0.0110808 2024-09-13T14:57:53.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:53.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:53.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:53.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:53.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:53.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:53.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:54.107 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:54.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:54.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:54.387 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:54.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:54 smithi159 ceph-mon[102519]: pgmap v1117: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/248928 objects misplaced (2.126%); 288 KiB/s, 4 objects/s recovering 2024-09-13T14:57:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:54 smithi159 ceph-mon[102519]: 12.5 scrub starts 2024-09-13T14:57:54.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:54.584 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:54 smithi042 ceph-mon[113926]: pgmap v1117: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/248928 objects misplaced (2.126%); 288 KiB/s, 4 objects/s recovering 2024-09-13T14:57:54.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:54 smithi042 ceph-mon[113926]: 12.5 scrub starts 2024-09-13T14:57:54.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:54 smithi042 ceph-mon[108570]: pgmap v1117: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/248928 objects misplaced (2.126%); 288 KiB/s, 4 objects/s recovering 2024-09-13T14:57:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:54 smithi042 ceph-mon[108570]: 12.5 scrub starts 2024-09-13T14:57:54.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:54.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 87566 87550 89.676 58.75 0.0240784 0.0111437 2024-09-13T14:57:54.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:54.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:54.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:54.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:54.992 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:54.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:54.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:55.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:55.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:55.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:55.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:55.474 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:55 smithi159 ceph-mon[102519]: 12.5 scrub ok 2024-09-13T14:57:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:55 smithi159 ceph-mon[102519]: 12.4 scrub starts 2024-09-13T14:57:55.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:55 smithi042 ceph-mon[113926]: 12.5 scrub ok 2024-09-13T14:57:55.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:55 smithi042 ceph-mon[113926]: 12.4 scrub starts 2024-09-13T14:57:55.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:55 smithi042 ceph-mon[108570]: 12.5 scrub ok 2024-09-13T14:57:55.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:55 smithi042 ceph-mon[108570]: 12.4 scrub starts 2024-09-13T14:57:55.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.532 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:55.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:55.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:55.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 88491 88475 89.1621 57.8125 0.0122129 0.0112096 2024-09-13T14:57:55.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:55.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:55.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:55.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:55.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:55.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.008 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.032 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:56.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:56.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:56.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:56.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:56.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:56 smithi159 ceph-mon[102519]: pgmap v1119: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/249951 objects misplaced (2.117%); 262 KiB/s, 4 objects/s recovering 2024-09-13T14:57:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:56 smithi159 ceph-mon[102519]: 12.4 scrub ok 2024-09-13T14:57:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:56 smithi159 ceph-mon[102519]: 12.0 scrub starts 2024-09-13T14:57:56.531 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:56.556 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[108570]: pgmap v1119: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/249951 objects misplaced (2.117%); 262 KiB/s, 4 objects/s recovering 2024-09-13T14:57:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[108570]: 12.4 scrub ok 2024-09-13T14:57:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[108570]: 12.0 scrub starts 2024-09-13T14:57:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[113926]: pgmap v1119: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 106 active+clean; 5.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 5292/249951 objects misplaced (2.117%); 262 KiB/s, 4 objects/s recovering 2024-09-13T14:57:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[113926]: 12.4 scrub ok 2024-09-13T14:57:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:56 smithi042 ceph-mon[113926]: 12.0 scrub starts 2024-09-13T14:57:56.657 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:56.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:56.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:56.717 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 89294 89278 88.5435 50.1875 0.0165817 0.0112877 2024-09-13T14:57:56.812 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:56.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:56.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:56.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:56.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:57.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:57.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.164 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:57.164 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:57:57.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:57.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:57.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:57.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:57.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:57.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:57.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:57.494 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:57.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:57.596 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:57.620 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:57.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 90128 90112 87.9744 52.125 0.00245704 0.0113588 2024-09-13T14:57:57.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:57.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:57.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:57.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:57.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:57.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:58.036 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:58.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:58.138 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:57:58.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:57:58.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:58.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:58.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.451 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:58.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:58 smithi159 ceph-mon[102519]: pgmap v1120: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 5292/251148 objects misplaced (2.107%); 171 KiB/s, 2 objects/s recovering 2024-09-13T14:57:58.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:58.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:58 smithi042 ceph-mon[113926]: pgmap v1120: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 5292/251148 objects misplaced (2.107%); 171 KiB/s, 2 objects/s recovering 2024-09-13T14:57:58.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:58 smithi042 ceph-mon[108570]: pgmap v1120: 109 pgs: 1 unknown, 1 active+recovering+undersized+remapped, 107 active+clean; 5.1 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 5292/251148 objects misplaced (2.107%); 171 KiB/s, 2 objects/s recovering 2024-09-13T14:57:58.681 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:57:58.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 91049 91033 87.5065 57.5625 0.00811926 0.0114224 2024-09-13T14:57:58.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:58.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:57:58.834 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:58.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:58.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:58.961 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:58.986 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:58.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:58.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:59.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:57:59.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:57:59.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:57:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:57:58.677+0000 7f8bec5d8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T14:57:59.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:57:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:57:58.677+0000 7f8bec5d8640 -1 osd.0 314 *** Got signal Terminated *** 2024-09-13T14:57:59.098 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:57:58 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[126291]: 2024-09-13T14:57:58.677+0000 7f8bec5d8640 -1 osd.0 314 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T14:57:59.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.141 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:57:59.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:57:59.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.263 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.365 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:57:59.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:57:59.489 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:57:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:57:59 smithi159 ceph-mon[102519]: osd.0 marked itself down and dead 2024-09-13T14:57:59.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:57:59.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:57:59 smithi042 ceph-mon[108570]: osd.0 marked itself down and dead 2024-09-13T14:57:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-13T14:57:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:57:59 smithi042 ceph-mon[113926]: osd.0 marked itself down and dead 2024-09-13T14:57:59.611 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:57:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:57:59.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 91483 91467 86.5917 27.125 0.00437178 0.0114542 2024-09-13T14:57:59.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:57:59.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:57:59.859 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:57:59.880 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:57:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:57:59.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:57:59.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:00.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:00.100 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:57:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:57:59] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.51.0" 2024-09-13T14:58:00.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.228 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.251 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:00.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:00.379 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:00.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:00 smithi159 ceph-mon[102519]: pgmap v1121: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/265203 objects misplaced (3.871%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:58:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:00 smithi159 ceph-mon[102519]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:58:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:00 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:58:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:00 smithi159 ceph-mon[102519]: osdmap e315: 8 total, 7 up, 7 in 2024-09-13T14:58:00.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[108570]: pgmap v1121: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/265203 objects misplaced (3.871%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:58:00.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[108570]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[108570]: osdmap e315: 8 total, 7 up, 7 in 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[113926]: pgmap v1121: 109 pgs: 2 active+recovering+undersized+remapped, 107 active+clean; 5.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/265203 objects misplaced (3.871%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[113926]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-13T14:58:00.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:00 smithi042 ceph-mon[113926]: osdmap e315: 8 total, 7 up, 7 in 2024-09-13T14:58:00.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:00.627 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:00.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 91899 91883 85.6875 26 0.00260265 0.0116199 2024-09-13T14:58:00.729 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:00.751 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:00.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:00.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:00.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:00.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:00.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:01.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:01.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:01.228 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:01.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:01.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:01 smithi042 ceph-mon[108570]: osdmap e316: 8 total, 7 up, 7 in 2024-09-13T14:58:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:01 smithi042 ceph-mon[108570]: osdmap e317: 8 total, 7 up, 7 in 2024-09-13T14:58:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:01 smithi042 ceph-mon[113926]: osdmap e316: 8 total, 7 up, 7 in 2024-09-13T14:58:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:01 smithi042 ceph-mon[113926]: osdmap e317: 8 total, 7 up, 7 in 2024-09-13T14:58:01.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:01 smithi042 podman[267206]: 2024-09-13 14:58:01.077458809 +0000 UTC m=+3.919870152 container died 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T14:58:01.362 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:01.403 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:01.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:01 smithi159 ceph-mon[102519]: osdmap e316: 8 total, 7 up, 7 in 2024-09-13T14:58:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:01 smithi159 ceph-mon[102519]: osdmap e317: 8 total, 7 up, 7 in 2024-09-13T14:58:01.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:01.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:01.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:01.632 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:01.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:01.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 92299 92283 84.7952 25 0.0178776 0.0117879 2024-09-13T14:58:01.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:01.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:01.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:01.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:01.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:01.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:02.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:02.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:02.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:02.169 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:02.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:02.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.298 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:02.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.400 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:02.423 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:02.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:02 smithi159 ceph-mon[102519]: pgmap v1124: 109 pgs: 14 peering, 14 stale+active+clean, 2 active+recovering+undersized+remapped, 79 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/269952 objects misplaced (3.803%); 2.5 MiB/s, 38 objects/s recovering 2024-09-13T14:58:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:02 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:58:02.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:02.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:02 smithi042 ceph-mon[113926]: pgmap v1124: 109 pgs: 14 peering, 14 stale+active+clean, 2 active+recovering+undersized+remapped, 79 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/269952 objects misplaced (3.803%); 2.5 MiB/s, 38 objects/s recovering 2024-09-13T14:58:02.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:02 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:58:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:02 smithi042 ceph-mon[108570]: pgmap v1124: 109 pgs: 14 peering, 14 stale+active+clean, 2 active+recovering+undersized+remapped, 79 active+clean; 5.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 10267/269952 objects misplaced (3.803%); 2.5 MiB/s, 38 objects/s recovering 2024-09-13T14:58:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:02 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 3 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-09-13T14:58:02.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:02.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:02.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.693 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:02.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 93507 93491 84.6604 75.5 0.0160647 0.0118066 2024-09-13T14:58:02.795 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:02.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:02.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:02.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:02.970 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:02.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.099 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:03.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:03.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:03.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:03.401 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:03 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 37179/276303 objects degraded (13.456%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:03.401 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:03 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 37179/276303 objects degraded (13.456%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:03.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.497 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:03.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:03.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:03.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:03.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 94687 94671 84.5043 73.75 0.0047539 0.0118283 2024-09-13T14:58:03.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:03 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 37179/276303 objects degraded (13.456%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:03.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:03.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:03.911 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:03.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:04.013 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:04.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:04.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:04.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.173 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.275 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:04.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:04.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:04.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:04.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:04 smithi042 ceph-mon[113926]: pgmap v1126: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.2 MiB/s wr, 51 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:58:04.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:04 smithi042 ceph-mon[108570]: pgmap v1126: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.2 MiB/s wr, 51 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:58:04.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:04.701 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:04.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:04.718 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 95736 95720 84.2374 65.5625 0.0300136 0.0118656 2024-09-13T14:58:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:04 smithi159 ceph-mon[102519]: pgmap v1126: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 3.2 MiB/s wr, 51 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 8.0 MiB/s, 128 objects/s recovering 2024-09-13T14:58:04.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:04.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:04.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:04.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:04.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:04.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:05.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:05.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:05.192 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.235 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:05.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:05.337 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:05.348 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:05 smithi042 podman[267206]: 2024-09-13 14:58:05.063764287 +0000 UTC m=+7.906175630 container cleanup 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-13T14:58:05.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:05 smithi042 bash[267206]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0 2024-09-13T14:58:05.379 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:05.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:05.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:05.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:05.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:05.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:05.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 96827 96811 84.0145 68.1875 0.00261454 0.0118971 2024-09-13T14:58:05.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:05.772 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:05.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:05.874 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:05.900 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:05.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:05.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:06.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:06.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:06.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:06.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:06.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:06.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:06.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.520 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:06.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[108570]: pgmap v1127: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[108570]: 12.1 scrub starts 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[113926]: pgmap v1127: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:58:06.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[113926]: 12.1 scrub starts 2024-09-13T14:58:06.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:06.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:06.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:06.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:06.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 97959 97943 83.8327 70.75 0.0101241 0.0119237 2024-09-13T14:58:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:06 smithi159 ceph-mon[102519]: pgmap v1127: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 14 peering, 1 active+recovering+undersized+remapped, 12 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 39 MiB/s wr, 616 op/s; 37179/276303 objects degraded (13.456%); 7098/276303 objects misplaced (2.569%); 5.0 MiB/s, 79 objects/s recovering 2024-09-13T14:58:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:06 smithi159 ceph-mon[102519]: 12.1 scrub starts 2024-09-13T14:58:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:06.792 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:06.835 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:06.936 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:06.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:06.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:06.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:07.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:07.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:07.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:07.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:07.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:07.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:07.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:07.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:07.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:07.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:07 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2024-09-13T14:58:07.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:07 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2024-09-13T14:58:07.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:07.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:07.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 99163 99147 83.7167 75.25 0.012485 0.0119401 2024-09-13T14:58:07.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:07 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 6 pgs peering) 2024-09-13T14:58:07.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:07.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:07.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:07.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:07.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:07.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:08.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:08.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:08.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:08.212 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:08.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:08.314 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:08.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:08.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:08.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:08.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:08.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:08.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:08 smithi042 ceph-mon[108570]: pgmap v1128: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 1 active+recovering+undersized+remapped, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 43130/278556 objects degraded (15.483%); 7098/278556 objects misplaced (2.548%); 4.1 MiB/s, 64 objects/s recovering 2024-09-13T14:58:08.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:08 smithi042 ceph-mon[108570]: 12.15 scrub starts 2024-09-13T14:58:08.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:08 smithi042 ceph-mon[113926]: pgmap v1128: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 1 active+recovering+undersized+remapped, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 43130/278556 objects degraded (15.483%); 7098/278556 objects misplaced (2.548%); 4.1 MiB/s, 64 objects/s recovering 2024-09-13T14:58:08.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:08 smithi042 ceph-mon[113926]: 12.15 scrub starts 2024-09-13T14:58:08.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:08.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:08.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 100230 100214 83.4895 66.6875 0.0269415 0.011972 2024-09-13T14:58:08.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:08 smithi159 ceph-mon[102519]: pgmap v1128: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 1 active+recovering+undersized+remapped, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 43130/278556 objects degraded (15.483%); 7098/278556 objects misplaced (2.548%); 4.1 MiB/s, 64 objects/s recovering 2024-09-13T14:58:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:08 smithi159 ceph-mon[102519]: 12.15 scrub starts 2024-09-13T14:58:08.842 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:08.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:08.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:08.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:08.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:08.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:09.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:09.125 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:09.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:09.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:09.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:09.409 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:09.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.539 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:09.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:09.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[113926]: 12.15 scrub ok 2024-09-13T14:58:09.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[113926]: 12.14 scrub starts 2024-09-13T14:58:09.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[113926]: 12.14 scrub ok 2024-09-13T14:58:09.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[108570]: 12.15 scrub ok 2024-09-13T14:58:09.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[108570]: 12.14 scrub starts 2024-09-13T14:58:09.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-mon[108570]: 12.14 scrub ok 2024-09-13T14:58:09.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:09.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:09.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:09.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 101332 101316 83.2972 68.875 0.0086142 0.0119995 2024-09-13T14:58:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:09 smithi159 ceph-mon[102519]: 12.15 scrub ok 2024-09-13T14:58:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:09 smithi159 ceph-mon[102519]: 12.14 scrub starts 2024-09-13T14:58:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:09 smithi159 ceph-mon[102519]: 12.14 scrub ok 2024-09-13T14:58:09.767 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:09.795 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:09.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:09.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:09.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:09.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:10.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:10.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:10.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:09] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-09-13T14:58:10.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:10.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:10.278 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:10.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:10.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:10 smithi042 ceph-mon[108570]: pgmap v1129: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s; 45162/292509 objects degraded (15.440%); 6556/292509 objects misplaced (2.241%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:58:10.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 45162/292509 objects degraded (15.440%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:10 smithi042 ceph-mon[113926]: pgmap v1129: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s; 45162/292509 objects degraded (15.440%); 6556/292509 objects misplaced (2.241%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:58:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 45162/292509 objects degraded (15.440%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:10.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:10.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 102310 102294 83.0092 61.125 0.0118241 0.0120411 2024-09-13T14:58:10.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:10 smithi159 ceph-mon[102519]: pgmap v1129: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 56 MiB/s wr, 900 op/s; 45162/292509 objects degraded (15.440%); 6556/292509 objects misplaced (2.241%); 6.5 MiB/s, 104 objects/s recovering 2024-09-13T14:58:10.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 45162/292509 objects degraded (15.440%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:10.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:10.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:10.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:10.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:10.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:11.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:11.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:11.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:11.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:11.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:11.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:11.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:11.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:11.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:11.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:11.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:11.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:11.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 103413 103397 82.8287 68.9375 0.0101274 0.0120678 2024-09-13T14:58:11.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:11.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:11.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:11.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:11.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:11.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:11.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.090 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.192 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:12.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:12.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:12 smithi042 ceph-mon[113926]: pgmap v1130: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 45507/295659 objects degraded (15.392%); 6556/295659 objects misplaced (2.217%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:58:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:12 smithi042 ceph-mon[113926]: 12.0 scrub starts 2024-09-13T14:58:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:12 smithi042 ceph-mon[108570]: pgmap v1130: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 45507/295659 objects degraded (15.392%); 6556/295659 objects misplaced (2.217%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:58:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:12 smithi042 ceph-mon[108570]: 12.0 scrub starts 2024-09-13T14:58:12.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:12.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:12.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:12.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 104534 104518 82.6671 70.0625 0.00477291 0.0120915 2024-09-13T14:58:12.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:12 smithi159 ceph-mon[102519]: pgmap v1130: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 54 MiB/s wr, 856 op/s; 45507/295659 objects degraded (15.392%); 6556/295659 objects misplaced (2.217%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:58:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:12 smithi159 ceph-mon[102519]: 12.0 scrub starts 2024-09-13T14:58:12.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:12.863 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:12.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:12.991 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:12.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:13.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:13.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:13.166 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:13 smithi042 podman[267398]: 2024-09-13 14:58:13.056631998 +0000 UTC m=+12.821003688 container remove 771c389bba871c92c3fd37576f0edd7ae1a9266003852ca133027f4f07780430 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T14:58:13.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:13.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:13.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:13.379 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:13.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:13.507 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:13.609 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:13.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:13.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T14:58:13.719328+0000 min lat: 0.00128182 max lat: 2.19784 avg lat: 0.0121061 2024-09-13T14:58:13.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T14:58:13.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 105731 105715 82.5688 74.8125 0.0248738 0.0121061 2024-09-13T14:58:13.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:13.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:13.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:13.911 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:13.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:13.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.054 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:14.148 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:14.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.281 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:14.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:14.441 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:14.467 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:14.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:14.569 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:14.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:14.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:14 smithi042 ceph-mon[113926]: pgmap v1131: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 7.0 MiB/s, 111 objects/s recovering 2024-09-13T14:58:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:14 smithi042 ceph-mon[113926]: 12.1 scrub starts 2024-09-13T14:58:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:14 smithi042 ceph-mon[108570]: pgmap v1131: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 7.0 MiB/s, 111 objects/s recovering 2024-09-13T14:58:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:14 smithi042 ceph-mon[108570]: 12.1 scrub starts 2024-09-13T14:58:14.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:14.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 106828 106812 82.3958 68.5625 0.00996169 0.0121302 2024-09-13T14:58:14.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:14 smithi159 ceph-mon[102519]: pgmap v1131: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 37 MiB/s wr, 592 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 7.0 MiB/s, 111 objects/s recovering 2024-09-13T14:58:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:14 smithi159 ceph-mon[102519]: 12.1 scrub starts 2024-09-13T14:58:14.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:14.870 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:14.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:14.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:14.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.121 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:15.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:15.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:15.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:15.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.609 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.628 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:58:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:58:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.628 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:107739 12.e 12:75f65b70:::benchmark_data_smithi042_260592_object107738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:58:15.544271+0000", 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.002458914, 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002500904, 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 107739 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:15.629 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544271+0000", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544271+0000", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544272+0000", 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.573e-06 2024-09-13T14:58:15.630 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544295+0000", 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2838000000000001e-05 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544308+0000", 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3132000000000001e-05 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:15.631 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544316+0000", 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1510000000000004e-06 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544380+0000", 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3856999999999995e-05 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544522+0000", 2024-09-13T14:58:15.632 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014153700000000001 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 7", 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544671+0000", 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014886200000000001 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:107738 12.19 e317/307 12:9b5b5555:::benchmark_data_smithi042_260592_object107737:head v 317'5800, mlcod=317'5798)", 2024-09-13T14:58:15.633 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:58:15.546456+0000", 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.000273663, 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00038720699999999998, 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546456+0000", 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.634 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546456+0000", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546456+0000", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3900000000000005e-07 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546482+0000", 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5678e-05 2024-09-13T14:58:15.635 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546490+0000", 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2430000000000005e-06 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:15.636 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546496+0000", 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5439999999999998e-06 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546629+0000", 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013316099999999999 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.546649+0000", 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9504000000000002e-05 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.637 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:107721 12.b 12:def45347:::benchmark_data_smithi042_260592_object107720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:58:15.524127+0000", 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0226021, 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022779793999999999, 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:15.638 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 107721 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524127+0000", 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524127+0000", 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:15.639 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524128+0000", 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7300000000000004e-07 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524154+0000", 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5463999999999999e-05 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524165+0000", 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1489e-05 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.640 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.524174+0000", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4840000000000004e-06 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.530588+0000", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064138240000000003 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.530709+0000", 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012108 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.641 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.530843+0000", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013434100000000001 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544850+0000", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014006819 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.544890+0000", 2024-09-13T14:58:15.642 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9674e-05 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.545031+0000", 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014094800000000001 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.545070+0000", 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9653999999999997e-05 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.643 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:107730 12.b 12:daed8f6b:::benchmark_data_smithi042_260592_object107729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:58:15.536394+0000", 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.01033513, 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010604631, 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 107730 2024-09-13T14:58:15.644 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536394+0000", 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536394+0000", 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:15.645 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536396+0000", 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.469e-06 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536420+0000", 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3628000000000001e-05 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.646 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536430+0000", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0164e-05 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536436+0000", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9660000000000001e-06 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.545127+0000", 2024-09-13T14:58:15.647 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086911479999999992 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.545238+0000", 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000110905 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,6", 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.545389+0000", 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015143200000000001 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:107731 12.18 12:1a811dc3:::benchmark_data_smithi042_260592_object107730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:58:15.536446+0000", 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.01028372, 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010624754, 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 107731 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.649 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536446+0000", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536446+0000", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536446+0000", 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4899999999999995e-07 2024-09-13T14:58:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536470+0000", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3703e-05 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536478+0000", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2530000000000004e-06 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536484+0000", 2024-09-13T14:58:15.651 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6450000000000002e-06 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536635+0000", 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000151299 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.536767+0000", 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132216 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T14:58:15.652 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.537108+0000", 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000340336 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:58:15.539228+0000", 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021206480000000002 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 5 2024-09-13T14:58:15.653 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:58:15.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:15.719 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 15 107928 107913 82.2301 68.8125 0.0205693 0.0121561 2024-09-13T14:58:15.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:15.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:15.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:15.879 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:15.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:15.981 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:16.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:16.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:16.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.148 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:16.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:16.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.437 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:16.564 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:16.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:16 smithi042 ceph-mon[113926]: pgmap v1132: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 47390/308907 objects degraded (15.341%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:16 smithi042 ceph-mon[108570]: pgmap v1132: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 47390/308907 objects degraded (15.341%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:16.666 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:16.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.706 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:16.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 109090 109074 82.1136 72.5625 0.0081111 0.012173 2024-09-13T14:58:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:16 smithi159 ceph-mon[102519]: pgmap v1132: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 24 MiB/s wr, 383 op/s; 47390/308907 objects degraded (15.341%); 6002/308907 objects misplaced (1.943%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 47390/308907 objects degraded (15.341%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:16.807 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.844 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:16.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:16.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:16.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:17.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.215 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:17.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:17.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.509 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:17.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:17.611 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:17.635 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:17.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 110272 110256 82.0155 73.875 0.00752889 0.0121867 2024-09-13T14:58:17.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:17.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:17.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:17.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:17.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:17.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:18.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:18.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:18.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:18.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:18.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:18.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T14:58:18.544 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.620 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:18 smithi042 ceph-mon[108570]: pgmap v1133: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 47729/311943 objects degraded (15.301%); 6002/311943 objects misplaced (1.924%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:18.621 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:18 smithi042 ceph-mon[113926]: pgmap v1133: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 47729/311943 objects degraded (15.301%); 6002/311943 objects misplaced (1.924%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:18.645 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 111416 111400 81.8917 71.5 0.0103525 0.0122058 2024-09-13T14:58:18.753 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:18 smithi159 ceph-mon[102519]: pgmap v1133: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 47729/311943 objects degraded (15.301%); 6002/311943 objects misplaced (1.924%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:58:18.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:18.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:18.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:18.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:18.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:19.028 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:19.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:19.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:19.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:19.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:19.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.353 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:19.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:19.454 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:19.507 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:19.545 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:19 smithi042 ceph-mon[108570]: 12.0 scrub starts 2024-09-13T14:58:19.546 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:19 smithi042 ceph-mon[113926]: 12.0 scrub starts 2024-09-13T14:58:19.546 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:19 smithi042 podman[268776]: 2024-09-13 14:58:19.389396866 +0000 UTC m=+0.381784516 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T14:58:19.546 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:19 smithi042 podman[268776]: 2024-09-13 14:58:19.541082296 +0000 UTC m=+0.533469941 container create d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T14:58:19.609 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:19.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:19.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:19.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 112443 112427 81.6858 64.1875 0.00965757 0.0122364 2024-09-13T14:58:19.734 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:19 smithi159 ceph-mon[102519]: 12.0 scrub starts 2024-09-13T14:58:19.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:19.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:19.847 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:19] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-09-13T14:58:19.874 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:19.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:19.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:19.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:20.004 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:20.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:20.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:20.099 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:20 smithi042 podman[268776]: 2024-09-13 14:58:20.055884775 +0000 UTC m=+1.048272418 container init d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T14:58:20.099 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:20 smithi042 podman[268776]: 2024-09-13 14:58:20.067632011 +0000 UTC m=+1.060019654 container start d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T14:58:20.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:20.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:20.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:20.310 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:20.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:20.371 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:20.380 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:58:20.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:20.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:20.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[108570]: pgmap v1134: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 49683/326052 objects degraded (15.238%); 5393/326052 objects misplaced (1.654%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3212260222' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[113926]: pgmap v1134: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 49683/326052 objects degraded (15.238%); 5393/326052 objects misplaced (1.654%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3212260222' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.487 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.488 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:20 smithi042 podman[268776]: 2024-09-13 14:58:20.356037751 +0000 UTC m=+1.348425405 container attach d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T14:58:20.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:20.556 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:58:20.556 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:58:20.556 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:58:20.557 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:58:20.557 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:20.557 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:20.557 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:20.563 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:58:20.563 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:20.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:58:20.602 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:20.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:20.640 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:58:20.640 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:20.675 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:58:20.675 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:20.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:20.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 113476 113460 81.489 64.5625 0.00925761 0.0122659 2024-09-13T14:58:20.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:20.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:20.749 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 14:58:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=infra.usagestats t=2024-09-13T14:58:20.342036246Z level=info msg="Usage stats are ready to report" 2024-09-13T14:58:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:20 smithi159 ceph-mon[102519]: pgmap v1134: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 49683/326052 objects degraded (15.238%); 5393/326052 objects misplaced (1.654%); 6.4 MiB/s, 101 objects/s recovering 2024-09-13T14:58:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3212260222' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:20.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:20.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:20.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:20.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:20.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:21.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:21.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:21.105 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:21.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:21.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:21.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:21.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:21.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.416 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:21.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:21.544 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:21.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:21.646 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:21.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:21.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 114631 114615 81.3832 72.1875 0.00730322 0.012282 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3138788826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 49683/326052 objects degraded (15.238%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: pgmap v1136: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 31 MiB/s wr, 489 op/s; 50005/329136 objects degraded (15.193%); 5393/329136 objects misplaced (1.639%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:58:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:21.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:21.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:21.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3138788826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:21.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 49683/326052 objects degraded (15.238%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: pgmap v1136: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 31 MiB/s wr, 489 op/s; 50005/329136 objects degraded (15.193%); 5393/329136 objects misplaced (1.639%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3138788826' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: osdmap e318: 8 total, 7 up, 7 in 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 49683/326052 objects degraded (15.238%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:21.851 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: pgmap v1136: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 31 MiB/s wr, 489 op/s; 50005/329136 objects degraded (15.193%); 5393/329136 objects misplaced (1.639%); 5.2 MiB/s, 83 objects/s recovering 2024-09-13T14:58:21.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:21.852 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:21.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:21.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:21.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:22.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.086 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.227 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:22.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:22.328 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.356 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:22.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:58:22.423 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:58:22.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:22.583 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:22.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:22.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:22.706 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:22 smithi042 ceph-mon[108570]: 12.1 scrub starts 2024-09-13T14:58:22.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:22 smithi042 ceph-mon[113926]: 12.1 scrub starts 2024-09-13T14:58:22.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.720 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 115800 115784 81.2897 73.0625 0.0263494 0.0122965 2024-09-13T14:58:22.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:22 smithi159 ceph-mon[102519]: 12.1 scrub starts 2024-09-13T14:58:22.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:22.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:22.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:22.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:23.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:23.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.147 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:23.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:23.324 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:22 smithi042 podman[268776]: 2024-09-13 14:58:22.977436742 +0000 UTC m=+3.969824394 container died d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T14:58:23.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:58:23.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:23.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:23.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:23 smithi042 ceph-mon[108570]: pgmap v1137: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:23.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:23 smithi042 ceph-mon[113926]: pgmap v1137: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:23.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:23.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:23.722 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.008 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 116888 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 81.165 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 34.4065 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 212.75 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 7.9375 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1298 2024-09-13T14:58:23.723 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 550.504 2024-09-13T14:58:23.724 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3404 2024-09-13T14:58:23.724 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 127 2024-09-13T14:58:23.724 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0123162 2024-09-13T14:58:23.724 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0377066 2024-09-13T14:58:23.724 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 2.19784 2024-09-13T14:58:23.725 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00128182 2024-09-13T14:58:23.727 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T14:58:23.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:23 smithi159 ceph-mon[102519]: pgmap v1137: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:23.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:23.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:23.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:23.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:23.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:23.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:24.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:24.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:24.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:24.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:24.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:24.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:24.349 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:24.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:24.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:58:24.450 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:24.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:24.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:24.600 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:24.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:24.634 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T14:58:24.681 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:24.684 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:24 smithi042 ceph-mon[113926]: 12.0 scrub starts 2024-09-13T14:58:24.685 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:24 smithi042 ceph-mon[108570]: 12.0 scrub starts 2024-09-13T14:58:24.685 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:24 smithi042 podman[269256]: 2024-09-13 14:58:24.599612086 +0000 UTC m=+1.649690531 container remove d9e14de46c53211b36bb07b4ff5f3b30f88eb46d1dcccb03258920bd5da07ced (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T14:58:24.685 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:24 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T14:58:24.685 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:24 smithi042 systemd[1]: Stopped Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T14:58:24.685 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 14:58:24 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service: Consumed 20min 22.763s CPU time. 2024-09-13T14:58:24.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:24.744 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T14:58:24.744 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T14:58:24.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:24 smithi159 ceph-mon[102519]: 12.0 scrub starts 2024-09-13T14:58:24.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:24.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:24.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:24.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:24.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:24.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:24.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:25.046 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:25.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:25.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:25.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:25.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:25.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:25.424 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:25.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:25.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:25.507 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:58:25.507 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:58:25.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:25.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:25.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:25.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:25 smithi042 ceph-mon[108570]: 12.1 scrub starts 2024-09-13T14:58:25.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:25 smithi042 ceph-mon[108570]: pgmap v1138: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:25.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:25 smithi042 ceph-mon[113926]: 12.1 scrub starts 2024-09-13T14:58:25.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:25 smithi042 ceph-mon[113926]: pgmap v1138: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:25.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:25.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:25 smithi159 ceph-mon[102519]: 12.1 scrub starts 2024-09-13T14:58:25.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:25 smithi159 ceph-mon[102519]: pgmap v1138: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s; 51833/341781 objects degraded (15.166%); 4767/341781 objects misplaced (1.395%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:25.815 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:25.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:25.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:25.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:25.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:25.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:26.083 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.249 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:26.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:26.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:26.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:26.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:26.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:26.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 51833/341781 objects degraded (15.166%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:26.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:26.797 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:26.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 51833/341781 objects degraded (15.166%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:26.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 51833/341781 objects degraded (15.166%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:26.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:26.934 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:26.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:26.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:27.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:27.061 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:27.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:27.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:27.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:27.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:27.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:27.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:27.568 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.591 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:27.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:27 smithi042 ceph-mon[108570]: pgmap v1139: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 52115/344727 objects degraded (15.118%); 4767/344727 objects misplaced (1.383%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:27.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:27 smithi042 ceph-mon[113926]: pgmap v1139: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 52115/344727 objects degraded (15.118%); 4767/344727 objects misplaced (1.383%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:27.694 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:27.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:27.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:27.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:27 smithi159 ceph-mon[102519]: pgmap v1139: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 74 MiB/s wr, 1.27k op/s; 52115/344727 objects degraded (15.118%); 4767/344727 objects misplaced (1.383%); 5.5 MiB/s, 88 objects/s recovering 2024-09-13T14:58:27.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:27.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:27.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:27.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:27.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:27.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:28.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:28.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:28.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:28.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:28.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:28.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:28.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:28.643 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:28.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:28.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:28.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:28.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:28.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:28.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:29.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:29.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:29.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:29.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:29.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:29.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:29.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:29.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.557 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:29.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:29.730 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:29.744 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0] out_osds: [4] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:58:29.745 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:29.745 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-09-13T14:58:29.745 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:58:29.745 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:58:29.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-09-13T14:58:29.832 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:29.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:29 smithi042 ceph-mon[108570]: pgmap v1140: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 51 MiB/s wr, 1.07k op/s; 51721/343389 objects degraded (15.062%); 4118/343389 objects misplaced (1.199%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:58:29.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:29 smithi042 ceph-mon[113926]: pgmap v1140: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 51 MiB/s wr, 1.07k op/s; 51721/343389 objects degraded (15.062%); 4118/343389 objects misplaced (1.199%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:58:29.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:29] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-13T14:58:29.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:29.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:29.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:29.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:29.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:29 smithi159 ceph-mon[102519]: pgmap v1140: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 7.0 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 51 MiB/s wr, 1.07k op/s; 51721/343389 objects degraded (15.062%); 4118/343389 objects misplaced (1.199%); 6.2 MiB/s, 99 objects/s recovering 2024-09-13T14:58:30.028 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:30.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.120 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:30.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:30.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:30.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:30.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:30.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:30.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:30.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.518 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:30.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.674 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:30.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 51721/343389 objects degraded (15.062%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:30.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:30.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:30.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:30.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:30.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:31.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 51721/343389 objects degraded (15.062%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 51721/343389 objects degraded (15.062%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:31.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:31.179 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:31.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:31.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T14:58:31.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:31.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:31.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:31.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:31.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:31.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:31.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:31.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.678 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:31.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:31.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:31.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:31.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[108570]: 12.0 scrub starts 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[108570]: pgmap v1141: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 100 B/s rd, 44 MiB/s wr, 1.05k op/s; 51350/340380 objects degraded (15.086%); 4118/340380 objects misplaced (1.210%); 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[113926]: 12.0 scrub starts 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[113926]: pgmap v1141: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 100 B/s rd, 44 MiB/s wr, 1.05k op/s; 51350/340380 objects degraded (15.086%); 4118/340380 objects misplaced (1.210%); 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:31.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:31.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:31 smithi159 ceph-mon[102519]: 12.0 scrub starts 2024-09-13T14:58:31.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:31 smithi159 ceph-mon[102519]: pgmap v1141: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.9 GiB data, 26 GiB used, 599 GiB / 626 GiB avail; 100 B/s rd, 44 MiB/s wr, 1.05k op/s; 51350/340380 objects degraded (15.086%); 4118/340380 objects misplaced (1.210%); 6.2 MiB/s, 98 objects/s recovering 2024-09-13T14:58:31.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T14:58:31.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.948 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:31.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:31.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:32.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:32.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:32.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:32.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:32.239 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:32.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:32.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:32.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:32.390 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:58:32.390 INFO:teuthology.orchestra.run.smithi159.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:58:32.390 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:58:32.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:32.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:32.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:32.646 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:32.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:32.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:32.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:32.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:32.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:32.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:32 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:32 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:33.041 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.064 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:33.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:33.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:32 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:33.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:32 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:33.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:32 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:33.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:32 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:33.165 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:33.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:33.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:33.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:33.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:33.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:33.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:33.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.594 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:33.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:33.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:33.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:33.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:33.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:33.865 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:33.966 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:33.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:33.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:33.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:34.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:33 smithi159 ceph-mon[102519]: pgmap v1142: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.25k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:58:34.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:33 smithi159 ceph-mon[102519]: 12.1 scrub starts 2024-09-13T14:58:34.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:34.096 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[108570]: pgmap v1142: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.25k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:58:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[108570]: 12.1 scrub starts 2024-09-13T14:58:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[113926]: pgmap v1142: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 85 B/s rd, 37 MiB/s wr, 1.25k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T14:58:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[113926]: 12.1 scrub starts 2024-09-13T14:58:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:58:34.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:34.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:34.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:34.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:34.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:34.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:34.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:34.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:34.650 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:34.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:34.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.775 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:34.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:34.911 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:34.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:34.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:34 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:34.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:34 smithi159 ceph-mon[102519]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:58:35.013 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.038 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:35.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[108570]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:58:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3144180598' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T14:58:35.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:34 smithi042 ceph-mon[113926]: osdmap e319: 8 total, 7 up, 7 in 2024-09-13T14:58:35.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:35.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:35.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:35.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:35.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:35.408 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:35.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:35.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:35.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:35.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:35.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:35.651 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:58:35.675 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:35.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:35.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:35.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:35.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:35.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:35 smithi159 ceph-mon[102519]: pgmap v1144: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 1.07k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:35.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 49194/327486 objects degraded (15.022%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:36.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:35 smithi042 ceph-mon[113926]: pgmap v1144: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 1.07k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:36.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 49194/327486 objects degraded (15.022%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:35 smithi042 ceph-mon[108570]: pgmap v1144: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.7 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 19 MiB/s wr, 1.07k op/s; 49194/327486 objects degraded (15.022%); 3386/327486 objects misplaced (1.034%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 49194/327486 objects degraded (15.022%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:36.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:36.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:36.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:36.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:36.338 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.375 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:36.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:36.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:36.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.501 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:36.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:36.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:36.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:36.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:36.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:36.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:36.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:36.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:36.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:36.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:36.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:36.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:37.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:37.137 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:37.265 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:37.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:37.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:37.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:37.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:37.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:37.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:37.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:37.695 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:37.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:37.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:37.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:37.973 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:37.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.079 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0] out_osds: [4] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:58:38.079 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:38.079 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-13T14:58:38.080 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T14:58:38.080 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T14:58:38.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-13T14:58:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:37 smithi042 ceph-mon[108570]: pgmap v1145: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 7.1 MiB/s wr, 1.07k op/s; 48559/319329 objects degraded (15.207%); 3386/319329 objects misplaced (1.060%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:38.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:37 smithi042 ceph-mon[113926]: pgmap v1145: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 7.1 MiB/s wr, 1.07k op/s; 48559/319329 objects degraded (15.207%); 3386/319329 objects misplaced (1.060%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:38.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:38.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:38.221 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:37 smithi159 ceph-mon[102519]: pgmap v1145: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.5 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 7.1 MiB/s wr, 1.07k op/s; 48559/319329 objects degraded (15.207%); 3386/319329 objects misplaced (1.060%); 7.8 MiB/s, 124 objects/s recovering 2024-09-13T14:58:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:38.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:38.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:38.409 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:38.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:38.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:38.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:38.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:38.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:38.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:38.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:38.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:38.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:38.949 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:38.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:39.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:39.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:39.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:39.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:39.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.367 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:39.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:39.494 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:39.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:39.596 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:39.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:39.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:39.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:39.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:39.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:39.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:39.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:39.987 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:39] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:58:40.115 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.155 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:40.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:40.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:40.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:40.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:40 smithi159 ceph-mon[102519]: pgmap v1146: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 46694/311241 objects degraded (15.003%); 2510/311241 objects misplaced (0.806%); 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:58:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:40 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.519 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:40.544 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[113926]: pgmap v1146: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 46694/311241 objects degraded (15.003%); 2510/311241 objects misplaced (0.806%); 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:58:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[108570]: pgmap v1146: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 46694/311241 objects degraded (15.003%); 2510/311241 objects misplaced (0.806%); 9.4 MiB/s, 150 objects/s recovering 2024-09-13T14:58:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:40 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:58:40.645 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:40.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:40.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:40.769 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:40.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:40.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:40.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:40.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:40.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:40.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.021 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:41.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:41.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:41.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:41.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:41.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:41.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:41.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:41.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:41.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:41.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:41 smithi159 ceph-mon[102519]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:58:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 46694/311241 objects degraded (15.003%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:41 smithi159 ceph-mon[102519]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:58:41.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:41.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[108570]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:58:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 46694/311241 objects degraded (15.003%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:41.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[108570]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:58:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[113926]: osdmap e320: 8 total, 7 up, 7 in 2024-09-13T14:58:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 46694/311241 objects degraded (15.003%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:41 smithi042 ceph-mon[113926]: osdmap e321: 8 total, 7 up, 7 in 2024-09-13T14:58:41.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:41.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:41.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:41.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:41.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:41.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:41.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:41.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:41.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:42.063 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:42.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:42.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:42.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:42.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:42.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:42.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:42.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:42.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:42.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:42 smithi159 ceph-mon[102519]: pgmap v1148: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 46372/308262 objects degraded (15.043%); 2510/308262 objects misplaced (0.814%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:58:42.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:42 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:58:42.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:42.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:42 smithi042 ceph-mon[108570]: pgmap v1148: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 46372/308262 objects degraded (15.043%); 2510/308262 objects misplaced (0.814%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:58:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:42 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:58:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:42 smithi042 ceph-mon[113926]: pgmap v1148: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.3 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 46372/308262 objects degraded (15.043%); 2510/308262 objects misplaced (0.814%); 6.3 MiB/s, 100 objects/s recovering 2024-09-13T14:58:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:42 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:58:42.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:42.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:42.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:42.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:42.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:42.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:42.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:42.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:42.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.050 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:43.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:43.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:43.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:43.410 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:43.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:43.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:43.559 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:43.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:43.683 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:43.785 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:43.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:43.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:43.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:43.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:43.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:43.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:44.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.098 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:44.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:44.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.376 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:44.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:44.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:44.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:44 smithi159 ceph-mon[102519]: pgmap v1150: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:44.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:44.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:44.536 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:44 smithi042 ceph-mon[108570]: pgmap v1150: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:44.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:44 smithi042 ceph-mon[113926]: pgmap v1150: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:44.588 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:44.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:44.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:44.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:44.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.789 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:44.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:44.891 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:44.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:44.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:44.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:45.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.044 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:45.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:45.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:45.171 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:45.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:45.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:45.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:45.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:45.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:45.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:45.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:45.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:45.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.730 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:45.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:45.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:45.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:45.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:45.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:46.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:46.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:46.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:46.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.229 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:46.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:46.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.448 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:46 smithi042 ceph-mon[108570]: pgmap v1151: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:46.448 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:46 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 44643/297423 objects degraded (15.010%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:46.448 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:46 smithi042 ceph-mon[113926]: pgmap v1151: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:46.448 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:46 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 44643/297423 objects degraded (15.010%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:46 smithi159 ceph-mon[102519]: pgmap v1151: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 6.0 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 912 op/s; 44643/297423 objects degraded (15.010%); 1790/297423 objects misplaced (0.602%); 10 MiB/s, 162 objects/s recovering 2024-09-13T14:58:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:46 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 44643/297423 objects degraded (15.010%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:46.507 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:58:46.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:46.549 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:58:46.550 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:58:46.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:46.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T14:58:46.593 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:58:46.593 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:58:46.631 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:58:46.631 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:58:46.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:46.668 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:58:46.668 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:58:46.701 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:58:46.701 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:58:46.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:46.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:46.735 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:58:46.735 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:58:46.766 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:58:46.767 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:58:46.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:46.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:46.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:46.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:46.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:46.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:46.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:47.118 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:47.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:47.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:47.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:47.366 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/983039352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:47.366 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:47 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:58:47.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:47.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:47.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/983039352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:47.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:47 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:58:47.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/983039352' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:58:47.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:47 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:58:47.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:47.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:47.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:47.773 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:47.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:47.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:47.946 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:47.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:48.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:48.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:48.467 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:48.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:48 smithi159 ceph-mon[102519]: pgmap v1152: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 40801/289023 objects degraded (14.117%); 4573/289023 objects misplaced (1.582%); 5.3 MiB/s, 84 objects/s recovering 2024-09-13T14:58:48.569 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:48.570 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:48 smithi042 ceph-mon[113926]: pgmap v1152: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 40801/289023 objects degraded (14.117%); 4573/289023 objects misplaced (1.582%); 5.3 MiB/s, 84 objects/s recovering 2024-09-13T14:58:48.571 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:48 smithi042 ceph-mon[108570]: pgmap v1152: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.9 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 925 op/s; 40801/289023 objects degraded (14.117%); 4573/289023 objects misplaced (1.582%); 5.3 MiB/s, 84 objects/s recovering 2024-09-13T14:58:48.630 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:48.732 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:48.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:48.880 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:48.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:48.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:49.026 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:49.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:49.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:49.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:49.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:49.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:49.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:49.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:49.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:49.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:49.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:49.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:58:49.618 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:58:49.715 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:49.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:49.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:49] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.51.0" 2024-09-13T14:58:49.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:49.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:49.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.000 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:50.102 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:50.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:50.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:50.357 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:50 smithi159 ceph-mon[102519]: pgmap v1153: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 39069/280920 objects degraded (13.908%); 4051/280920 objects misplaced (1.442%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:58:50.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:50.564 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:50 smithi042 ceph-mon[108570]: pgmap v1153: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 39069/280920 objects degraded (13.908%); 4051/280920 objects misplaced (1.442%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:58:50.564 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:50 smithi042 ceph-mon[113926]: pgmap v1153: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.7 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 39069/280920 objects degraded (13.908%); 4051/280920 objects misplaced (1.442%); 8.1 MiB/s, 130 objects/s recovering 2024-09-13T14:58:50.618 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:50.646 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:50.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:50.775 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:50.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:50.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:50.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:50.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:51.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:51.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:51.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:51.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:51.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.220 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:51.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:51.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:51.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:51.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:51 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 39069/280920 objects degraded (13.908%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:51.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:51.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:51 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 39069/280920 objects degraded (13.908%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:51.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:51.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:51.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:51.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:51 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 39069/280920 objects degraded (13.908%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T14:58:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:58:51.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:51.603 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:51.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:51.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:58:51.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.762 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:51.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:51.863 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:51.899 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:58:51.899 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T14:58:51.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T14:58:51.899 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213820 10.17 10:ee44b033:::benchmark_data_smithi042_191918_object90227:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52181939800005, 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.784238448, 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.900 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213820 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371943+0000", 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371944+0000", 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.595e-06 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371950+0000", 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.716e-06 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371966+0000", 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5879000000000002e-05 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371976+0000", 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0655e-05 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372653+0000", 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00067709599999999997 2024-09-13T14:58:51.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372666+0000", 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2221e-05 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.143834+0000", 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.771168766 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.143978+0000", 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014373700000000001 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.144467+0000", 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00048926199999999997 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.155915+0000", 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011447327 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.155986+0000", 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1088999999999996e-05 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156039+0000", 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2808e-05 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156052+0000", 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3793e-05 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156102+0000", 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9953000000000001e-05 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156181+0000", 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8551000000000001e-05 2024-09-13T14:58:51.907 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213886 10.17 10:eef5bd04:::benchmark_data_smithi042_191918_object90293:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52177105299995, 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7933409569999998, 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.908 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213886 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.909 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371991+0000", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-07 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.371994+0000", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4619999999999999e-06 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372001+0000", 2024-09-13T14:58:51.910 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5830000000000001e-06 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372008+0000", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.156e-06 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372698+0000", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000690615 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372703+0000", 2024-09-13T14:58:51.911 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4680000000000003e-06 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156224+0000", 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7835214420000001 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156310+0000", 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5722000000000004e-05 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.912 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.156714+0000", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000404127 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161136+0000", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044215519999999996 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161150+0000", 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4524e-05 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.913 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165147+0000", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039972719999999996 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165210+0000", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2856999999999998e-05 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165247+0000", 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6650999999999999e-05 2024-09-13T14:58:51.914 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165332+0000", 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5041000000000001e-05 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213947 10.17 10:ebf6dfbb:::benchmark_data_smithi042_191918_object90354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52174700800003, 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.793580567, 2024-09-13T14:58:51.915 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213947 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.916 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372015+0000", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0900000000000002e-07 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372017+0000", 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.052e-06 2024-09-13T14:58:51.917 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372024+0000", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2639999999999997e-06 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372029+0000", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1309999999999997e-06 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372732+0000", 2024-09-13T14:58:51.918 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00070308600000000001 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372738+0000", 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2230000000000004e-06 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.158652+0000", 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7859140469999999 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.919 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.158784+0000", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132275 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.159222+0000", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000437637 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161209+0000", 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001987366 2024-09-13T14:58:51.920 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165086+0000", 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003876586 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165456+0000", 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00037029700000000001 2024-09-13T14:58:51.921 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165487+0000", 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0728000000000001e-05 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165518+0000", 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0593000000000003e-05 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.922 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165596+0000", 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7873000000000005e-05 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213949 10.17 10:eaeba952:::benchmark_data_smithi042_191918_object90356:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:58:51.923 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52172743100004, 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.799407583, 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213949 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.924 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:58:51.925 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372035+0000", 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6300000000000001e-07 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372037+0000", 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6500000000000001e-06 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.926 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372047+0000", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0339999999999999e-05 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372052+0000", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1590000000000001e-06 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372770+0000", 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00071784300000000002 2024-09-13T14:58:51.927 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372776+0000", 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3539999999999996e-06 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161242+0000", 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7884659549999999 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.928 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161355+0000", 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011312299999999999 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.161609+0000", 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025334000000000001 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.929 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165384+0000", 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037758560000000002 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165402+0000", 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7918999999999999e-05 2024-09-13T14:58:51.930 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169316+0000", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039136450000000003 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171256+0000", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019401290000000001 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171320+0000", 2024-09-13T14:58:51.931 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3584000000000007e-05 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171442+0000", 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122323 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213960 10.17 10:ede664c3:::benchmark_data_smithi042_191918_object90367:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:58:51.932 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52170437799998, 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.807093547, 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213960 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.933 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372058+0000", 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4400000000000001e-07 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.934 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372060+0000", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.685e-06 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372066+0000", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1709999999999999e-06 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372071+0000", 2024-09-13T14:58:51.935 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8960000000000002e-06 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372811+0000", 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00074009900000000003 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372819+0000", 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1869999999999997e-06 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.936 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163146+0000", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7903273820000001 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163218+0000", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1724000000000006e-05 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.163517+0000", 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00029936699999999998 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.937 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165607+0000", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020893909999999999 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165647+0000", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0482000000000003e-05 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171492+0000", 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0058448980000000003 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.938 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179017+0000", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075249970000000003 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179067+0000", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9592000000000002e-05 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179151+0000", 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4332000000000005e-05 2024-09-13T14:58:51.939 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213988 10.17 10:edab4be2:::benchmark_data_smithi042_191918_object90395:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52168426699996, 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8073287489999998, 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.940 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213988 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.941 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372078+0000", 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3500000000000002e-07 2024-09-13T14:58:51.942 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372080+0000", 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7999999999999999e-06 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.943 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372085+0000", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4609999999999999e-06 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372089+0000", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0600000000000001e-06 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372847+0000", 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00075744500000000004 2024-09-13T14:58:51.944 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372853+0000", 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8830000000000003e-06 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.945 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:51.946 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:51.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165685+0000", 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7928323150000001 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.947 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.165756+0000", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0901000000000004e-05 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.166010+0000", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000253945 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169332+0000", 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033222019999999998 2024-09-13T14:58:51.948 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171543+0000", 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0022114330000000001 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179213+0000", 2024-09-13T14:58:51.949 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076700029999999999 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179264+0000", 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0346000000000002e-05 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179298+0000", 2024-09-13T14:58:51.950 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3772999999999997e-05 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179406+0000", 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108847 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.952 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:213999 10.17 10:e927b90b:::benchmark_data_smithi042_191918_object90406:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52166750000004, 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.812156871, 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 213999 2024-09-13T14:58:51.953 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372094+0000", 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.954 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372095+0000", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9900000000000002e-07 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372096+0000", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.544e-06 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372102+0000", 2024-09-13T14:58:51.955 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2630000000000003e-06 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372106+0000", 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4719999999999997e-06 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372911+0000", 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00080517800000000004 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.956 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372919+0000", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3159999999999999e-06 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167500+0000", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7945812290000001 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167599+0000", 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9600999999999997e-05 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.957 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.167791+0000", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000191218 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171561+0000", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003770027 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179472+0000", 2024-09-13T14:58:51.958 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007911019 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.183988+0000", 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045164380000000002 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184143+0000", 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015473900000000001 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.959 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184181+0000", 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7799000000000002e-05 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184251+0000", 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0729000000000007e-05 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.960 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214016 10.17 10:ec50e907:::benchmark_data_smithi042_191918_object90423:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52164852700002, 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.833510296, 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214016 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.961 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372113+0000", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372114+0000", 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6600000000000002e-07 2024-09-13T14:58:51.962 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372116+0000", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.782e-06 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372122+0000", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4150000000000004e-06 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372127+0000", 2024-09-13T14:58:51.963 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6149999999999999e-06 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372940+0000", 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00081335999999999995 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372946+0000", 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1399999999999997e-06 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.964 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169379+0000", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7964327500000001 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169460+0000", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1364000000000002e-05 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.169676+0000", 2024-09-13T14:58:51.965 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021588999999999999 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171567+0000", 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018909560000000001 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184041+0000", 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012474071 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.966 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184100+0000", 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9222999999999999e-05 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205461+0000", 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.021360494000000001 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205515+0000", 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.38e-05 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.967 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205624+0000", 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010907 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214058 10.17 10:eb527693:::benchmark_data_smithi042_191918_object90465:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.521630114, 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8338334629999999, 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.968 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214058 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.969 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372132+0000", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372134+0000", 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:58:51.970 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372141+0000", 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.063e-06 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372146+0000", 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5310000000000003e-06 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.971 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372996+0000", 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085073799999999997 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373003+0000", 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9179999999999997e-06 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171583+0000", 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7985794429999999 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.972 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171651+0000", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8609000000000007e-05 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.171850+0000", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019853899999999999 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179160+0000", 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007310464 2024-09-13T14:58:51.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189042+0000", 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0098813229999999991 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189161+0000", 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011959399999999999 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205674+0000", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016513116000000001 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205809+0000", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013482699999999999 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.205965+0000", 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015622100000000001 2024-09-13T14:58:51.975 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214091 10.17 10:ef747aa2:::benchmark_data_smithi042_191918_object90498:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52161114700004, 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8340415399999999, 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.976 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214091 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372151+0000", 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2399999999999999e-07 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372153+0000", 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7430000000000001e-06 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.978 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372158+0000", 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5480000000000001e-06 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372163+0000", 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9779999999999996e-06 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373060+0000", 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00089612399999999999 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373064+0000", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.985e-06 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173248+0000", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8001843879999999 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173364+0000", 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116449 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.173684+0000", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031991700000000002 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179424+0000", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057400389999999997 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189094+0000", 2024-09-13T14:58:51.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096700840000000007 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189287+0000", 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000192797 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206016+0000", 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016728845999999999 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.982 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206077+0000", 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1397999999999997e-05 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206192+0000", 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011492 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.983 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214096 10.17 10:e8b3be78:::benchmark_data_smithi042_191918_object90503:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.52159341200002, 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.834227032, 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.984 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214096 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:58:51.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372169+0000", 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.34e-07 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372171+0000", 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.699e-06 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.986 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372178+0000", 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.08e-06 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.372183+0000", 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.541e-06 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373085+0000", 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00090176500000000001 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.373091+0000", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0850000000000002e-06 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176017+0000", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.802925707 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176142+0000", 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000125635 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.176503+0000", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00036053799999999998 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179430+0000", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029274700000000002 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189346+0000", 2024-09-13T14:58:51.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0099155010000000002 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192704+0000", 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033585160000000002 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206245+0000", 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013540942 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.990 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206298+0000", 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2794999999999999e-05 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206396+0000", 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7423999999999994e-05 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214183 10.17 10:e94dc7de:::benchmark_data_smithi042_191918_object90590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:51.991 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.45132963000003, 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.764709037, 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:51.992 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214183 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.993 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442432+0000", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442434+0000", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3120000000000001e-06 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442442+0000", 2024-09-13T14:58:51.994 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3610000000000001e-06 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442452+0000", 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.008e-05 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442457+0000", 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9819999999999999e-06 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.995 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442498+0000", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0735000000000002e-05 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.442504+0000", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6039999999999998e-06 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179503+0000", 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7369989829999999 2024-09-13T14:58:51.996 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179571+0000", 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7988e-05 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.179886+0000", 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031447400000000003 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:51.997 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184110+0000", 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042242970000000001 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189409+0000", 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052988510000000003 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199205+0000", 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097961820000000005 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.998 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206437+0000", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072323140000000001 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.206483+0000", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5970000000000002e-05 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207141+0000", 2024-09-13T14:58:51.999 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00065790399999999995 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214185 10.17 10:e975695b:::benchmark_data_smithi042_191918_object90592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.45042065300004, 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7640301379999999, 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.000 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214185 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.001 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443341+0000", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443342+0000", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3300000000000002e-07 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443345+0000", 2024-09-13T14:58:52.002 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.225e-06 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443353+0000", 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0190000000000006e-06 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443357+0000", 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3100000000000002e-06 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.003 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443396+0000", 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8965999999999999e-05 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.443401+0000", 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8239999999999999e-06 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.004 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.181721+0000", 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7383201939999999 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.181835+0000", 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113781 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.182076+0000", 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000240635 2024-09-13T14:58:52.005 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.186916+0000", 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048399890000000003 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189459+0000", 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025427190000000001 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.006 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199247+0000", 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097884879999999997 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207190+0000", 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079428859999999997 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207248+0000", 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.766e-05 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.007 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207371+0000", 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012380900000000001 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214190 10.17 10:ecea52bb:::benchmark_data_smithi042_191918_object90597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.44317713099997, 2024-09-13T14:58:52.008 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7569722080000001, 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214190 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.009 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450585+0000", 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450586+0000", 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3009999999999999e-06 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.010 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450591+0000", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9710000000000003e-06 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450600+0000", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7900000000000005e-06 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450605+0000", 2024-09-13T14:58:52.011 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8500000000000002e-06 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450658+0000", 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3009e-05 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.450668+0000", 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8630000000000003e-06 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.012 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184291+0000", 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7336238420000001 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184365+0000", 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3829000000000001e-05 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:52.013 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.184662+0000", 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000296345 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189177+0000", 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045157039999999997 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189500+0000", 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00032273399999999998 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.014 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199282+0000", 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097823219999999995 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207411+0000", 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081288769999999996 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.015 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207459+0000", 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7642999999999998e-05 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207557+0000", 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8128000000000006e-05 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.016 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214243 10.17 10:ee61d6bb:::benchmark_data_smithi042_191918_object90650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.33105921900005, 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.645319298, 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214243 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562703+0000", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562704+0000", 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.637e-06 2024-09-13T14:58:52.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562709+0000", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3460000000000004e-06 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562718+0000", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8799999999999997e-06 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562723+0000", 2024-09-13T14:58:52.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9450000000000001e-06 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562779+0000", 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5989999999999998e-05 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.562789+0000", 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0151e-05 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.186965+0000", 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.62417641 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.187065+0000", 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100016 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.187302+0000", 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023705800000000001 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189364+0000", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020619779999999999 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192310+0000", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029459080000000001 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199326+0000", 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0070158210000000002 2024-09-13T14:58:52.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207595+0000", 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082688569999999992 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.207642+0000", 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6981000000000003e-05 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208022+0000", 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00038032000000000002 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45032.0:214298 10.17 10:ebc9c796:::benchmark_data_smithi042_191918_object90705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 536.07327576199998, 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3877340249999999, 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.024 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45032", 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214298 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820486+0000", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820487+0000", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2899999999999999e-06 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820492+0000", 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3059999999999999e-06 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820501+0000", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5279999999999992e-06 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820507+0000", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3530000000000002e-06 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820561+0000", 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4466000000000003e-05 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:55.820571+0000", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5510000000000005e-06 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189546+0000", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3689755100000001 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.189656+0000", 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000109638 2024-09-13T14:58:52.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5,6", 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.190003+0000", 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034756400000000002 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192406+0000", 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002402141 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.029 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.192450+0000", 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4297e-05 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.199916+0000", 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007466215 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208069+0000", 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0081527119999999995 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208121+0000", 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2392000000000003e-05 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:49:57.208220+0000", 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.9061999999999994e-05 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.031 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86125 12.11 12:8a0d8b36:::benchmark_data_smithi042_260592_object86124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151484+0000", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 59.742278065999997, 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3037125899999999, 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86125 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3399999999999999e-07 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151505+0000", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0979e-05 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151513+0000", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3180000000000001e-06 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151518+0000", 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4040000000000003e-06 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152398+0000", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00087995399999999996 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152404+0000", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1680000000000001e-06 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416477+0000", 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2640725930000001 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416584+0000", 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010674 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416736+0000", 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015215300000000001 2024-09-13T14:58:52.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422773+0000", 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060373429999999997 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445727+0000", 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022953881999999998 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453059+0000", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0073320290000000003 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453124+0000", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5162999999999999e-05 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.455196+0000", 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020723299999999998 2024-09-13T14:58:52.038 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86135 12.11 12:8c7021b7:::benchmark_data_smithi042_260592_object86134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151525+0000", 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 59.742237058000001, 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3064091740000001, 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.039 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86135 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T14:58:52.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4000000000000002e-07 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151544+0000", 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8833000000000001e-05 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151551+0000", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7279999999999998e-06 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151556+0000", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3129999999999998e-06 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152425+0000", 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00086871300000000002 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152431+0000", 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.144e-06 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.417930+0000", 2024-09-13T14:58:52.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2654990859999999 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418028+0000", 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7684999999999997e-05 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418152+0000", 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012413100000000001 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422904+0000", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047515930000000001 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445774+0000", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022870715 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456350+0000", 2024-09-13T14:58:52.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010575721 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456409+0000", 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9438000000000002e-05 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.457934+0000", 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001524634 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.046 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86235 12.11 12:8a3e8959:::benchmark_data_smithi042_260592_object86234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151562+0000", 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 59.742199556999999, 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3274169330000001, 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86235 2024-09-13T14:58:52.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151563+0000", 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2300000000000002e-07 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151587+0000", 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3813000000000001e-05 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151593+0000", 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.844e-06 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151599+0000", 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2009999999999998e-06 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152452+0000", 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085325899999999998 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152458+0000", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.049e-06 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422991+0000", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.270532545 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423159+0000", 2024-09-13T14:58:52.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016851999999999999 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423348+0000", 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018931899999999999 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.428067+0000", 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047184879999999998 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445827+0000", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017759664000000001 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474222+0000", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.028395482999999999 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474279+0000", 2024-09-13T14:58:52.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7204999999999999e-05 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.478979+0000", 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047001200000000003 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86275 12.11 12:8d556199:::benchmark_data_smithi042_260592_object86274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151605+0000", 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 59.742156985999998, 2024-09-13T14:58:52.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3296465079999999, 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86275 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T14:58:52.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4999999999999998e-07 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151627+0000", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1914999999999999e-05 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151634+0000", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6479999999999997e-06 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151639+0000", 2024-09-13T14:58:52.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.578e-06 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152479+0000", 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00084047799999999997 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152485+0000", 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.207e-06 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424679+0000", 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721941349999999 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424837+0000", 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015725900000000001 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:58:52.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.425047+0000", 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000210137 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.429698+0000", 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046507639999999999 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445901+0000", 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016203077999999999 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479037+0000", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033136456000000002 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479191+0000", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015432 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.481251+0000", 2024-09-13T14:58:52.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020600829999999999 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T14:58:52.062 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:58:52.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:52.087 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:52.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:52.188 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:52.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:52.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:52.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:52.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:52.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:52.445 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:52.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:52.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:52 smithi159 ceph-mon[102519]: pgmap v1154: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 38696/277044 objects degraded (13.967%); 3881/277044 objects misplaced (1.401%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:58:52.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:52.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:52 smithi042 ceph-mon[108570]: pgmap v1154: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 38696/277044 objects degraded (13.967%); 3881/277044 objects misplaced (1.401%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:58:52.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:52 smithi042 ceph-mon[113926]: pgmap v1154: 109 pgs: 1 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.6 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 1.04k op/s; 38696/277044 objects degraded (13.967%); 3881/277044 objects misplaced (1.401%); 6.8 MiB/s, 108 objects/s recovering 2024-09-13T14:58:52.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.627 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:52.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:52.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.755 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:52.857 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:52.882 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:52.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:52.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:52.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:53.110 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:53.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:53.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:53.265 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:53.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:53.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:53.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:53 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:58:53.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:53.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:53.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:53.569 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T14:58:53.570 INFO:teuthology.orchestra.run.smithi042.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T14:58:53.570 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T14:58:53.570 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:53 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:58:53.570 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:53 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:58:53.657 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.683 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:53.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:53.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:53.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:53.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:53.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:54.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:54.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:54.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.210 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:54.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:54.348 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:54.450 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:54 smithi159 ceph-mon[102519]: pgmap v1155: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 8.6 MiB/s, 137 objects/s recovering 2024-09-13T14:58:54.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:54.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:54 smithi042 ceph-mon[108570]: pgmap v1155: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 8.6 MiB/s, 137 objects/s recovering 2024-09-13T14:58:54.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:54 smithi042 ceph-mon[113926]: pgmap v1155: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 8.6 MiB/s, 137 objects/s recovering 2024-09-13T14:58:54.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:54.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:54.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:54.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:54.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:54.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:54.865 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:54.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:54.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:55.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:55.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.017 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:55.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:55.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.159 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:55.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:55.190 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:55.261 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:55.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:55.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:55.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.411 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:55.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:55.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:55.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:55.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:55.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:55.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:55.931 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:55.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:56.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:56.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:56.088 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T14:58:56.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:56.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:56.306 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:56.322 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[113926]: pgmap v1156: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 36547/263358 objects degraded (13.877%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:58:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[113926]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:58:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[108570]: pgmap v1156: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 36547/263358 objects degraded (13.877%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:58:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:56 smithi042 ceph-mon[108570]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:58:56.424 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:56.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:56 smithi159 ceph-mon[102519]: pgmap v1156: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.4 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 945 op/s; 36547/263358 objects degraded (13.877%); 3271/263358 objects misplaced (1.242%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:56.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 36547/263358 objects degraded (13.877%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T14:58:56.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:56 smithi159 ceph-mon[102519]: osdmap e322: 8 total, 7 up, 7 in 2024-09-13T14:58:56.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:56.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:56.513 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:58:56.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:56.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:56.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:56.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:56.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:56.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:56.845 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:56.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:56.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:56.989 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:56.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:56.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:57.091 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.131 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:57.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:57.232 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:57.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:57.391 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:58:57.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:58:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:57 smithi159 ceph-mon[102519]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:58:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:57 smithi159 ceph-mon[102519]: pgmap v1159: 109 pgs: 1 active+clean+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 35852/253866 objects degraded (14.122%); 2591/253866 objects misplaced (1.021%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:57.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:57.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:57.680 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:57 smithi042 ceph-mon[108570]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:58:57.680 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:57 smithi042 ceph-mon[108570]: pgmap v1159: 109 pgs: 1 active+clean+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 35852/253866 objects degraded (14.122%); 2591/253866 objects misplaced (1.021%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:57.681 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:57 smithi042 ceph-mon[113926]: osdmap e323: 8 total, 7 up, 7 in 2024-09-13T14:58:57.681 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:57 smithi042 ceph-mon[113926]: pgmap v1159: 109 pgs: 1 active+clean+remapped, 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.2 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 35852/253866 objects degraded (14.122%); 2591/253866 objects misplaced (1.021%); 5.6 MiB/s, 89 objects/s recovering 2024-09-13T14:58:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.696 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:57.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:57.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:57.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:57.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:57.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:57.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:57.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:58.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.098 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:58.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:58.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:58.200 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:58:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:58:58.200 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T14:58:58.200 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:58:58.230 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:58.332 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:58:58.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:58:58.462 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:58:58.491 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:58:58.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:58.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:58.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:58.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:58.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:58.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3087700090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:58 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3087700090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:58 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:58.868 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3087700090' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.868 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:58 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:58:58.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:58.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:58.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:58:59.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:59.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:58:59.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:59.167 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:59.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T14:58:59.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:59.311 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:59.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:59.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:59.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:59.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:59.510 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:58:59.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:58:59.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:58:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:58:59.675 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:58:59.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:58:59.789 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:59 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:58:59.790 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:59 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:59.790 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:59.790 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:59 smithi159 ceph-mon[102519]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:58:59.790 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:58:59 smithi159 ceph-mon[102519]: pgmap v1161: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 30842/247125 objects degraded (12.480%); 5849/247125 objects misplaced (2.367%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:58:59.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:58:59.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:58:59.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:59.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:59.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[108570]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:58:59.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[108570]: pgmap v1161: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 30842/247125 objects degraded (12.480%); 5849/247125 objects misplaced (2.367%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:58:59] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.51.0" 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[113926]: osdmap e324: 8 total, 7 up, 7 in 2024-09-13T14:58:59.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:58:59 smithi042 ceph-mon[113926]: pgmap v1161: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 901 op/s; 30842/247125 objects degraded (12.480%); 5849/247125 objects misplaced (2.367%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:58:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:58:59.852 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:58:59.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:58:59.922 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0] out_osds: [4] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:58:59.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:58:59.922 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-09-13T14:58:59.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd in 4 2024-09-13T14:58:59.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:00.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.010 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:00.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:00.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.141 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:00.242 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:00.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:00.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:00.437 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:00.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:00.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.564 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:00.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:00.666 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:00.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 30842/247125 objects degraded (12.480%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:00.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:00.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:00.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:00.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 30842/247125 objects degraded (12.480%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:00.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 30842/247125 objects degraded (12.480%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T14:59:00.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:00.971 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:00.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:01.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:01.098 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:01.200 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:01.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:01.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:01.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:01.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:01.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:01.462 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:01.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:01.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:01.571 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:01.571 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:59:01.571 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:59:01.571 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:59:01.571 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:01.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:01.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.623 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:01.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:01.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:01.764 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:01 smithi042 ceph-mon[113926]: pgmap v1162: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 30524/243804 objects degraded (12.520%); 5696/243804 objects misplaced (2.336%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:59:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:01 smithi042 ceph-mon[108570]: pgmap v1162: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 30524/243804 objects degraded (12.520%); 5696/243804 objects misplaced (2.336%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:59:01.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:01.906 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:01.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:01.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:01.912 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:01 smithi159 ceph-mon[102519]: pgmap v1162: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 1 creating+peering, 28 active+undersized, 17 active+undersized+degraded, 61 active+clean; 5.0 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 30524/243804 objects degraded (12.520%); 5696/243804 objects misplaced (2.336%); 4.7 MiB/s, 75 objects/s recovering 2024-09-13T14:59:02.008 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.037 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:02.138 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.250 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T14:59:02.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:02.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:02.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:02.456 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:02.536 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:02.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:02.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:02.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:02.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:02.732 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:02.834 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:02.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:02.964 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:02.990 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:02.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:03.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:03.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:03.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:03.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:03.296 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:03.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:03.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:03.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:03.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:03.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:03 smithi159 ceph-mon[102519]: pgmap v1163: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:59:03.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:03.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:03.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:03.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:03.948 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:03.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:04.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:04.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:03 smithi042 ceph-mon[113926]: pgmap v1163: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:59:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:03 smithi042 ceph-mon[108570]: pgmap v1163: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.34k op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 5.7 MiB/s, 91 objects/s recovering 2024-09-13T14:59:04.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:04.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:04.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:04.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:04.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:04.455 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:04.480 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:04.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:04.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:04.621 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:04.723 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:04.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:04.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:04.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:04.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:04.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:04.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:04 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:59:04.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.000 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:04 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:59:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:04 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:59:05.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:05.132 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:05.145 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T14:59:05.234 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.255 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.357 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:05.395 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:05.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:05.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:05.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:05.748 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:05.748 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:59:05.748 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:59:05.748 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:59:05.748 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:156728 11.c e279/271 11:39b4de92:::benchmark_data_smithi042_223320_object37722:head v 279'9863, mlcod=279'9860)", 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:53:56.903086+0000", 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 308.22251094199999, 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0083244530000000008, 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903086+0000", 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.749 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903086+0000", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903085+0000", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903094+0000", 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6670000000000002e-06 2024-09-13T14:59:05.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903110+0000", 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6637e-05 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903118+0000", 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9160000000000001e-06 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.751 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903166+0000", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8245999999999999e-05 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.903191+0000", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4864000000000001e-05 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.911336+0000", 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0081446290000000005 2024-09-13T14:59:05.752 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:53:56.911411+0000", 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4857000000000005e-05 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.753 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:202515 11.c e291/271 11:329d7bac:::benchmark_data_smithi042_223320_object83509:head v 291'12640, mlcod=291'12638)", 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:55:10.544108+0000", 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 234.58148958300001, 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010373402, 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544108+0000", 2024-09-13T14:59:05.754 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544108+0000", 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544106+0000", 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:05.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544115+0000", 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8599999999999999e-06 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544138+0000", 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3054999999999999e-05 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544147+0000", 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6670000000000002e-06 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.756 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544195+0000", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7652999999999999e-05 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.544219+0000", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4005000000000001e-05 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.554438+0000", 2024-09-13T14:59:05.757 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010219684999999999 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:10.554481+0000", 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2734000000000001e-05 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:223647 11.c e293/271 11:3e581c07:::benchmark_data_smithi042_223320_object104641:head v 293'13991, mlcod=293'13989)", 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:55:43.377130+0000", 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 201.7484676, 2024-09-13T14:59:05.758 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0082233129999999995, 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377130+0000", 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377130+0000", 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.759 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377128+0000", 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377138+0000", 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6360000000000006e-06 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.760 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377154+0000", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5532e-05 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377160+0000", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4130000000000003e-06 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377203+0000", 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3387999999999999e-05 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.761 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.377223+0000", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9187000000000001e-05 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.385323+0000", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0081006910000000001 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:43.385353+0000", 2024-09-13T14:59:05.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9779000000000001e-05 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:236071 11.c e295/271 11:34382825:::benchmark_data_smithi042_223320_object117065:head v 295'14793, mlcod=295'14790)", 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:56:03.274075+0000", 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 181.851522195, 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0091120209999999997, 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:05.763 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274075+0000", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274075+0000", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274074+0000", 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:05.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274083+0000", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0040000000000005e-06 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274098+0000", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5293000000000001e-05 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.274107+0000", 2024-09-13T14:59:05.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.405e-06 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.276343+0000", 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022368319999999998 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.276373+0000", 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9422999999999999e-05 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:05.766 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.283110+0000", 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067371619999999997 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:56:03.283187+0000", 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7025000000000002e-05 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:05.767 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:05.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:05.792 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:05.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:05.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:05.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:05.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:05 smithi159 ceph-mon[102519]: pgmap v1164: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:59:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 28838/231063 objects degraded (12.481%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:06.071 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:05 smithi042 ceph-mon[108570]: pgmap v1164: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:59:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 28838/231063 objects degraded (12.481%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:05 smithi042 ceph-mon[113926]: pgmap v1164: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.7 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 942 op/s; 28838/231063 objects degraded (12.481%); 5314/231063 objects misplaced (2.300%); 3.1 MiB/s, 49 objects/s recovering 2024-09-13T14:59:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 28838/231063 objects degraded (12.481%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:06.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.197 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:06.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:06.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:06.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:06.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:06.579 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:06.606 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:06.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:06.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:06.831 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:06.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:06.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:06.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:06.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:07.062 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:07.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:07.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:07.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:07.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:07.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:07.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:07.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:07.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:07.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:07.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.691 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T14:59:07.704 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:07.835 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:07.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:07.852 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:59:07.852 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:59:07.852 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:07.883 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:59:07.884 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:07.884 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:07 smithi042 ceph-mon[113926]: pgmap v1165: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 28143/221664 objects degraded (12.696%); 5141/221664 objects misplaced (2.319%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:59:07.884 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:07 smithi042 ceph-mon[108570]: pgmap v1165: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 28143/221664 objects degraded (12.696%); 5141/221664 objects misplaced (2.319%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:59:07.929 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:59:07.930 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:59:07.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:07.957 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:59:07.958 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:07.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:07.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:07.973 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:07.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:59:07.991 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:07 smithi159 ceph-mon[102519]: pgmap v1165: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.5 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 28143/221664 objects degraded (12.696%); 5141/221664 objects misplaced (2.319%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T14:59:08.030 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:59:08.030 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:08.063 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:59:08.063 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:08.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:08.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:08.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:08.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:08.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:08.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:08.368 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:08.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.494 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:08.596 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.621 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:08.723 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:08.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.750 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:08.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:08.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2242736319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:08.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:08.896 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:08.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:08.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.032 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:09.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2242736319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:09.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2242736319' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:09.100 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T14:59:09.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.148 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:09.149 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:09.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.198 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:09.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:09.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:09.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:09.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.567 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:09.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:09.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:09.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:09.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:09.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:09.809 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:09.810 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:09.810 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:09 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:59:09.810 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:09 smithi159 ceph-mon[102519]: pgmap v1166: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 26788/214677 objects degraded (12.478%); 4585/214677 objects misplaced (2.136%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:59:09.841 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:09.867 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:09.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:09.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:09.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:10.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:10.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:09 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:59:10.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:09 smithi042 ceph-mon[108570]: pgmap v1166: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 26788/214677 objects degraded (12.478%); 4585/214677 objects misplaced (2.136%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:59:10.058 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:09] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.51.0" 2024-09-13T14:59:10.058 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:09 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:59:10.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:09 smithi042 ceph-mon[113926]: pgmap v1166: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.4 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 26788/214677 objects degraded (12.478%); 4585/214677 objects misplaced (2.136%); 2.9 MiB/s, 46 objects/s recovering 2024-09-13T14:59:10.114 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:10.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.163 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:10.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:10.264 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:10.311 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:10.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:10.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:10.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:10.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:10.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T14:59:10.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:10.622 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:10.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:10.707 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 26788/214677 objects degraded (12.478%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:10.707 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 26788/214677 objects degraded (12.478%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:10.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:10.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:10.826 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:10.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:10.895 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:10.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:10.955 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 26788/214677 objects degraded (12.478%), 17 pgs degraded, 46 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:10.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.040 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:11.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.044 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:11.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:11.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:59:11.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:11.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:11.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:11.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:11.397 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:11.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.551 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:11.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T14:59:11.340+0000 7fcad2a32640 -1 mon.a@0(leader).osd e324 definitely_dead 0 2024-09-13T14:59:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:11.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:11.681 INFO:teuthology.orchestra.run.smithi042.stderr:marked in osd.4. 2024-09-13T14:59:11.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:11.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:11.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:11.913 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[113926]: pgmap v1167: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 26460/211392 objects degraded (12.517%); 4421/211392 objects misplaced (2.091%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:59:11.913 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4258548354' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:11.913 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:11.913 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[108570]: pgmap v1167: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 26460/211392 objects degraded (12.517%); 4421/211392 objects misplaced (2.091%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:59:11.914 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4258548354' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:11.914 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:11 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:11.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:11 smithi159 ceph-mon[102519]: pgmap v1167: 109 pgs: 1 active+clean+remapped, 1 active+recovering+undersized+remapped, 28 active+undersized, 17 active+undersized+degraded, 62 active+clean; 4.3 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 992 op/s; 26460/211392 objects degraded (12.517%); 4421/211392 objects misplaced (2.091%); 2.4 MiB/s, 39 objects/s recovering 2024-09-13T14:59:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:11 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4258548354' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:11 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-13T14:59:12.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:12.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:12.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:12.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:12.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:12.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:12.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:12.504 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:12.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:12.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:12.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:12.734 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:12.755 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:12.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:59:12.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[108570]: osdmap e325: 8 total, 7 up, 8 in 2024-09-13T14:59:12.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[108570]: osdmap e326: 8 total, 7 up, 8 in 2024-09-13T14:59:12.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:59:12.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[113926]: osdmap e325: 8 total, 7 up, 8 in 2024-09-13T14:59:12.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:12 smithi042 ceph-mon[113926]: osdmap e326: 8 total, 7 up, 8 in 2024-09-13T14:59:12.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:12.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:12.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:12.980 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:12 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-13T14:59:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:12 smithi159 ceph-mon[102519]: osdmap e325: 8 total, 7 up, 8 in 2024-09-13T14:59:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:12 smithi159 ceph-mon[102519]: osdmap e326: 8 total, 7 up, 8 in 2024-09-13T14:59:13.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:13.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:13.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.124 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:13.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.352 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:13.375 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:13.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:13.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:13.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:13.569 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:13.600 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:13.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:13.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:13.727 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.767 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:13.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:13.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:13 smithi042 ceph-mon[113926]: pgmap v1169: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:59:13.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:13 smithi042 ceph-mon[113926]: osdmap e327: 8 total, 7 up, 8 in 2024-09-13T14:59:13.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:13 smithi042 ceph-mon[108570]: pgmap v1169: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:59:13.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:13 smithi042 ceph-mon[108570]: osdmap e327: 8 total, 7 up, 8 in 2024-09-13T14:59:13.868 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:13.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:13.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:13.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:13.956 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:13 smithi159 ceph-mon[102519]: pgmap v1169: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 3.3 MiB/s, 53 objects/s recovering 2024-09-13T14:59:13.956 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:13 smithi159 ceph-mon[102519]: osdmap e327: 8 total, 7 up, 8 in 2024-09-13T14:59:14.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:14.055 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:14.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:14.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:14.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:14.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:14.251 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:14.252 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:14.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:14.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:14.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:14.451 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-09-13T14:59:14.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:14.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:14.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:14.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:14.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:14.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:14.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:14.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:14.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:14.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:14.966 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:14 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:59:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:14 smithi159 ceph-mon[102519]: 12.e starting backfill to osd.4 from (0'0,0'0] MAX to 324'10478 2024-09-13T14:59:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:14 smithi159 ceph-mon[102519]: osdmap e328: 8 total, 7 up, 8 in 2024-09-13T14:59:15.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:15.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:15.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T14:59:15.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:59:15.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[108570]: 12.e starting backfill to osd.4 from (0'0,0'0] MAX to 324'10478 2024-09-13T14:59:15.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[108570]: osdmap e328: 8 total, 7 up, 8 in 2024-09-13T14:59:15.084 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:59:15.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[113926]: 12.e starting backfill to osd.4 from (0'0,0'0] MAX to 324'10478 2024-09-13T14:59:15.085 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:14 smithi042 ceph-mon[113926]: osdmap e328: 8 total, 7 up, 8 in 2024-09-13T14:59:15.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:15.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:15.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:15.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:15.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:15.318 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:15.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:15.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:15.522 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:15.559 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:15.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:15.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:15.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:15.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:15.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:15.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:15.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:15.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:15.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:15.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:15 smithi159 ceph-mon[102519]: pgmap v1172: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:59:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 25081/200256 objects degraded (12.524%), 15 pgs degraded, 44 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:15.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:15 smithi159 ceph-mon[102519]: osdmap e329: 8 total, 7 up, 8 in 2024-09-13T14:59:16.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:16.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[113926]: pgmap v1172: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 25081/200256 objects degraded (12.524%), 15 pgs degraded, 44 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[113926]: osdmap e329: 8 total, 7 up, 8 in 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[108570]: pgmap v1172: 109 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 active+clean+remapped, 28 active+undersized, 15 active+undersized+degraded, 61 active+clean; 4.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 800 op/s; 25081/200256 objects degraded (12.524%); 3915/200256 objects misplaced (1.955%); 2.6 MiB/s, 42 objects/s recovering 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 25081/200256 objects degraded (12.524%), 15 pgs degraded, 44 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:15 smithi042 ceph-mon[108570]: osdmap e329: 8 total, 7 up, 8 in 2024-09-13T14:59:16.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:16.227 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:16.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:16.328 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:16.356 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:16.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:16.500 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:16.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:16.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:16.740 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T14:59:16.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:16.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:16.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:16.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:16.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:16 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4241388792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:16.940 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:16 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:16.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:16.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:17.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.086 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:16 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4241388792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:17.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:16 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:17.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:16 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4241388792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:17.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:16 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T14:59:17.188 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:17.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:17.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:17.265 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:17.265 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:17.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:17.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:17.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:17.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:17.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:17.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:17.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:17.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:17 smithi159 ceph-mon[102519]: pgmap v1175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 10 active+clean+remapped, 16 active+undersized, 12 active+undersized+degraded, 65 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 28538/195684 objects degraded (14.584%); 3815/195684 objects misplaced (1.950%); 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:59:17.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:17 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:17.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:17 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:17.752 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:17 smithi159 ceph-mon[102519]: osdmap e330: 8 total, 7 up, 8 in 2024-09-13T14:59:17.764 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[113926]: pgmap v1175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 10 active+clean+remapped, 16 active+undersized, 12 active+undersized+degraded, 65 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 28538/195684 objects degraded (14.584%); 3815/195684 objects misplaced (1.950%); 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[113926]: osdmap e330: 8 total, 7 up, 8 in 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[108570]: pgmap v1175: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 4 peering, 10 active+clean+remapped, 16 active+undersized, 12 active+undersized+degraded, 65 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 380 op/s; 28538/195684 objects degraded (14.584%); 3815/195684 objects misplaced (1.950%); 13 B/s, 2 keys/s, 1 objects/s recovering 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T14:59:17.783 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:17 smithi042 ceph-mon[108570]: osdmap e330: 8 total, 7 up, 8 in 2024-09-13T14:59:17.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:17.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:17.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:17.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:17.998 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:18.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:18.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:18.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:18.207 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T14:59:18.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:18.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.294 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:18.344 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:18.345 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:59:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:18.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:18.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:18.514 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:18.542 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:18.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:18.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:18.841 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:18.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:18.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:18.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:18.992 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:18.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:18.994 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:18 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/654868973' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:18.994 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:18.994 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:18 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.093 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:19.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/654868973' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:19.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/654868973' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:19.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:18 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.131 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:19.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:19.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:19.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:19.388 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:19.452 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T14:59:19.452 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T14:59:19.456 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-13T14:59:19.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T14:59:19.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.521 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:19.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:19.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.651 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:19.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:19 smithi159 ceph-mon[102519]: 12.10 scrub starts 2024-09-13T14:59:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:19 smithi159 ceph-mon[102519]: pgmap v1177: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 65 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 30986/189312 objects degraded (16.368%); 8242/189312 objects misplaced (4.354%); 9.8 MiB/s, 2 keys/s, 159 objects/s recovering 2024-09-13T14:59:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:19 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:19 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:19.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:19 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.754 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:19.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:19.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[108570]: 12.10 scrub starts 2024-09-13T14:59:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[108570]: pgmap v1177: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 65 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 30986/189312 objects degraded (16.368%); 8242/189312 objects misplaced (4.354%); 9.8 MiB/s, 2 keys/s, 159 objects/s recovering 2024-09-13T14:59:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:19.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:19] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.51.0" 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[113926]: 12.10 scrub starts 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[113926]: pgmap v1177: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 65 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 30986/189312 objects degraded (16.368%); 8242/189312 objects misplaced (4.354%); 9.8 MiB/s, 2 keys/s, 159 objects/s recovering 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T14:59:19.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:19 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:19.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:19.931 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:19.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:19.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:20.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:20.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:20.164 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:20.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:20.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:20.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:20.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:20.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.475 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:20.475 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:59:20.475 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.475 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174823 12.f 12:f03c7f63:::benchmark_data_smithi042_260592_object57932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.475 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.465244+0000", 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00702296, 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.007047401, 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174823 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.476 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465244+0000", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465244+0000", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465243+0000", 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465249+0000", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4820000000000002e-06 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465259+0000", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9189999999999994e-06 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465263+0000", 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.357e-06 2024-09-13T14:59:20.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465309+0000", 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6292000000000003e-05 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465435+0000", 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000125427 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:20.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465547+0000", 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112083 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.472271+0000", 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067240340000000003 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174824 12.f 12:fd3106a4:::benchmark_data_smithi042_260592_object57933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.480 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.465271+0000", 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0069961310000000001, 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0070725650000000003, 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174824 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.481 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465271+0000", 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465271+0000", 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465271+0000", 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465273+0000", 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4370000000000001e-06 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465284+0000", 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0502e-05 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465289+0000", 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9300000000000002e-06 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.468999+0000", 2024-09-13T14:59:20.484 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003710302 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.469098+0000", 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8805e-05 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.469250+0000", 2024-09-13T14:59:20.485 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015271600000000001 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174825 12.4 12:25e62a22:::benchmark_data_smithi042_260592_object57934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.465293+0000", 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0069739930000000004, 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0070908179999999996, 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.486 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174825 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465293+0000", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465293+0000", 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465293+0000", 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465295+0000", 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5610000000000001e-06 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465306+0000", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0214000000000001e-05 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465310+0000", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6029999999999998e-06 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465354+0000", 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3597999999999998e-05 2024-09-13T14:59:20.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465443+0000", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9708000000000002e-05 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.465891+0000", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00044723000000000001 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.469092+0000", 2024-09-13T14:59:20.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032016269999999999 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.470798+0000", 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0017058830000000001 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174834 12.f 12:fafb7663:::benchmark_data_smithi042_260592_object57943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.471718+0000", 2024-09-13T14:59:20.491 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00054882399999999995, 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00071182799999999996, 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174834 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471718+0000", 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.492 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471718+0000", 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471717+0000", 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471723+0000", 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.896e-06 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471739+0000", 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5625e-05 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.471746+0000", 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5279999999999998e-06 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.472308+0000", 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00056189200000000003 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.472410+0000", 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010225099999999999 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.495 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174814 12.1 12:8235aa89:::benchmark_data_smithi042_260592_object57923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.455942+0000", 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.016325374, 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.016523612999999999, 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.496 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174814 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455942+0000", 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.497 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455942+0000", 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455941+0000", 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455947+0000", 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8069999999999996e-06 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455961+0000", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.466e-05 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.455968+0000", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0469999999999996e-06 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.456022+0000", 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3591999999999998e-05 2024-09-13T14:59:20.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.456263+0000", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00024131700000000001 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.456716+0000", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00045313200000000002 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.462127+0000", 2024-09-13T14:59:20.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0054111419999999999 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.462159+0000", 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1350000000000003e-05 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:174817 12.1 12:877842f4:::benchmark_data_smithi042_260592_object57926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:20.457891+0000", 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.01437573, 2024-09-13T14:59:20.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.014618126, 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174817 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457891+0000", 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457891+0000", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457890+0000", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457896+0000", 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0870000000000003e-06 2024-09-13T14:59:20.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457908+0000", 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1578e-05 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.457914+0000", 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5270000000000004e-06 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:20.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.459607+0000", 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001692673 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.459754+0000", 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014713000000000001 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,7", 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.460173+0000", 2024-09-13T14:59:20.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041908499999999998 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.462281+0000", 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0021079900000000001 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:20.463123+0000", 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00084172499999999998 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:20.506 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:59:20.507 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 6 2024-09-13T14:59:20.507 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.508 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:20.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:20.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:20.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:20.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:20.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.792 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:20.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:20.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:20.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:20.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:20.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:20 smithi159 ceph-mon[102519]: 12.10 scrub ok 2024-09-13T14:59:20.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 30986/189312 objects degraded (16.368%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:21.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:21.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:21.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:20 smithi042 ceph-mon[108570]: 12.10 scrub ok 2024-09-13T14:59:21.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 30986/189312 objects degraded (16.368%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:21.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:20 smithi042 ceph-mon[113926]: 12.10 scrub ok 2024-09-13T14:59:21.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 30986/189312 objects degraded (16.368%), 15 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:21.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:21.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:21.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T14:59:21.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:21.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:21.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:21.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:21.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:21.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:21.517 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:21.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:21.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:21.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:21.704 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:21.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:21.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:21.846 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:21.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:21.882 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:21.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:21.900 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:21 smithi159 ceph-mon[102519]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 30689/186327 objects degraded (16.471%); 8142/186327 objects misplaced (4.370%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:21.900 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:21.983 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:22.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:22.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:21 smithi042 ceph-mon[108570]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 30689/186327 objects degraded (16.471%); 8142/186327 objects misplaced (4.370%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:21 smithi042 ceph-mon[113926]: pgmap v1178: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 30689/186327 objects degraded (16.471%); 8142/186327 objects misplaced (4.370%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:22.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:22.143 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:22.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:22.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:22.273 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:22.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:22.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:22.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:22.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:22.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:22.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:22.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:22.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:22.648 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:22.749 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:22.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:22.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:22.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:22.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:22.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:22 smithi159 ceph-mon[102519]: osdmap e331: 8 total, 7 up, 8 in 2024-09-13T14:59:23.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:23.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:23.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:22 smithi042 ceph-mon[113926]: osdmap e331: 8 total, 7 up, 8 in 2024-09-13T14:59:23.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:22 smithi042 ceph-mon[108570]: osdmap e331: 8 total, 7 up, 8 in 2024-09-13T14:59:23.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:23.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.292 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:23.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:23.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:23.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.552 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:23.628 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:23.629 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:59:23.629 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:59:23.629 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:59:23.629 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:23.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:23.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:23.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:23.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:23.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:23.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:23 smithi159 ceph-mon[102519]: pgmap v1180: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:24.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:23 smithi159 ceph-mon[102519]: osdmap e332: 8 total, 7 up, 8 in 2024-09-13T14:59:24.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:23 smithi042 ceph-mon[108570]: pgmap v1180: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:24.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:23 smithi042 ceph-mon[108570]: osdmap e332: 8 total, 7 up, 8 in 2024-09-13T14:59:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:23 smithi042 ceph-mon[113926]: pgmap v1180: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%); 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T14:59:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:23 smithi042 ceph-mon[113926]: osdmap e332: 8 total, 7 up, 8 in 2024-09-13T14:59:24.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:24.219 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:24.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:24.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:24.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T14:59:24.460 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:24.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:24.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:24.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:24.696 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:24.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.775 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:24.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:24.854 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:24.876 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:24.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:24.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:25.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:25.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:25.304 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:25.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:25.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:25.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.560 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:25.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:25.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:25.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:25.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:25 smithi042 ceph-mon[113926]: pgmap v1182: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%) 2024-09-13T14:59:25.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:25 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 29208/175122 objects degraded (16.679%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:25 smithi042 ceph-mon[108570]: pgmap v1182: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%) 2024-09-13T14:59:25.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:25 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 29208/175122 objects degraded (16.679%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:25.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:25.948 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:25.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:25.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:25.976 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:25 smithi159 ceph-mon[102519]: pgmap v1182: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 194 op/s; 29208/175122 objects degraded (16.679%); 7081/175122 objects misplaced (4.043%) 2024-09-13T14:59:25.976 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:25 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 29208/175122 objects degraded (16.679%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:26.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:26.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:26.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:26.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:26.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:26.349 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:26.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:26.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:26.487 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:26.589 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:26.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:26.659 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213300 10.16 10.2fc051b6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.86801703499998, 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.004492741, 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.660 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213300 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786534+0000", 2024-09-13T14:59:26.661 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786533+0000", 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786539+0000", 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4999999999999996e-06 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.662 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786551+0000", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1389000000000001e-05 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786557+0000", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5359999999999998e-06 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789016+0000", 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002459265 2024-09-13T14:59:26.663 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789023+0000", 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.443e-06 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791016+0000", 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.001992759 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.664 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791027+0000", 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1364e-05 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213303 10.16 10.11a593b6 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.86798483200005, 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0044890209999999, 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.665 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213303 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.666 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786566+0000", 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786569+0000", 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9100000000000001e-06 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.667 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786575+0000", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7579999999999998e-06 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.786580+0000", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3489999999999999e-06 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789039+0000", 2024-09-13T14:59:26.668 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002458876 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.789043+0000", 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3710000000000002e-06 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791048+0000", 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0020046730000001 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.669 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791055+0000", 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4549999999999998e-06 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213349 10.16 10.8c4fc176 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.83210106299998, 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96863077099999995, 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.670 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213349 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.671 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822450+0000", 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822449+0000", 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822454+0000", 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5799999999999999e-06 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.672 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822467+0000", 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2092e-05 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822472+0000", 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8259999999999998e-06 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.673 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822516+0000", 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3659000000000003e-05 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.822523+0000", 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3300000000000001e-06 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791073+0000", 2024-09-13T14:59:26.674 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96854951199999995 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791081+0000", 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.011e-06 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213416 10.16 10.6199b816 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.675 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.75804444400001, 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89460015299999995, 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213416 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.676 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896507+0000", 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896505+0000", 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.677 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896517+0000", 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2329e-05 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896530+0000", 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2782999999999999e-05 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.678 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896537+0000", 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1969999999999999e-06 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896745+0000", 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00020745599999999999 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.896752+0000", 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0679999999999999e-06 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.679 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791100+0000", 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89434780599999997 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791107+0000", 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0949999999999998e-06 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.680 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213480 10.16 10.2728f336 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.69820742399997, 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.83479029299999996, 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.681 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213480 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:59:26.682 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956344+0000", 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.683 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956343+0000", 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956347+0000", 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2989999999999998e-06 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.684 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956356+0000", 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6829999999999997e-06 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956362+0000", 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2210000000000002e-06 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.685 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956625+0000", 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00026299400000000001 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.956635+0000", 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0190999999999999e-05 2024-09-13T14:59:26.686 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791127+0000", 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.83449121999999998 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791134+0000", 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4240000000000004e-06 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.687 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213511 10.16 10.8fe7b616 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.66452642399997, 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80113405299999996, 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.688 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213511 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990025+0000", 2024-09-13T14:59:26.689 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990023+0000", 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.690 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.691 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.691 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990030+0000", 2024-09-13T14:59:26.691 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0700000000000001e-06 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990043+0000", 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2337000000000001e-05 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990049+0000", 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6769999999999998e-06 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.692 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990108+0000", 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8445e-05 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:52.990116+0000", 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2160000000000006e-06 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.693 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.694 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791151+0000", 2024-09-13T14:59:26.694 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80103517700000004 2024-09-13T14:59:26.694 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.694 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791159+0000", 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.515e-06 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.695 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45032.0:213544 10.16 10.adf34f56 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e229)", 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 573.62835696399998, 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76498887800000004, 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45032", 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2228204185", 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213544 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.696 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026194+0000", 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.697 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026193+0000", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026197+0000", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9570000000000004e-06 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026206+0000", 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4130000000000006e-06 2024-09-13T14:59:26.698 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026212+0000", 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0920000000000003e-06 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026252+0000", 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0210000000000003e-05 2024-09-13T14:59:26.699 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.026259+0000", 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2080000000000003e-06 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791176+0000", 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76491660800000005 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.700 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:49:53.791183+0000", 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1219999999999997e-06 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:88849 12.15 12:a9e3f313:::benchmark_data_smithi042_260592_object88848:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e314)", 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:56.165821+0000", 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 90.488730337000007, 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14299231500000001, 2024-09-13T14:59:26.701 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 88849 2024-09-13T14:59:26.702 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165821+0000", 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165821+0000", 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.703 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165819+0000", 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165873+0000", 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3390999999999998e-05 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.704 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165881+0000", 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9000000000000006e-06 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165885+0000", 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.143e-06 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.165932+0000", 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7561000000000001e-05 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.705 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.166079+0000", 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014651200000000001 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.166174+0000", 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5024000000000004e-05 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.706 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.167427+0000", 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0012533620000000001 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.174054+0000", 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066268630000000002 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.308760+0000", 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13470555000000001 2024-09-13T14:59:26.707 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.308791+0000", 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1714000000000002e-05 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:56.308813+0000", 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1766000000000001e-05 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.708 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:101932 12.f 12:f0db0dc8:::benchmark_data_smithi042_260592_object101931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:10.293997+0000", 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 76.360553785999997, 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.137777081, 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.709 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 101932 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.293997+0000", 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.293997+0000", 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.710 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.293997+0000", 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294059+0000", 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2875999999999999e-05 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.711 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294072+0000", 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2751999999999999e-05 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294101+0000", 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8649e-05 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294152+0000", 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1567999999999998e-05 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.712 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294299+0000", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014683099999999999 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294473+0000", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000173714 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.300761+0000", 2024-09-13T14:59:26.713 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0062880480000000001 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.300784+0000", 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2863000000000001e-05 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.431717+0000", 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13093343499999999 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.714 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.431749+0000", 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2020000000000002e-05 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.431774+0000", 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5165999999999999e-05 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.715 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:101933 12.f 12:fd0a6fa7:::benchmark_data_smithi042_260592_object101932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e317)", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:10.294114+0000", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 76.360437661999995, 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.14399719799999999, 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 101933 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.716 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294114+0000", 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294114+0000", 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294113+0000", 2024-09-13T14:59:26.717 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294167+0000", 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3865e-05 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294180+0000", 2024-09-13T14:59:26.718 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3124e-05 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.294185+0000", 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.356e-06 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.297797+0000", 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036113719999999998 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.719 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.297953+0000", 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015660699999999999 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.298099+0000", 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014526299999999999 2024-09-13T14:59:26.720 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.300949+0000", 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028499490000000001 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.304028+0000", 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030794260000000001 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.721 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.438041+0000", 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.13401301199999999 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.438079+0000", 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7580999999999998e-05 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.722 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:10.438111+0000", 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2157999999999999e-05 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:161108 12.5 12:a15692a0:::benchmark_data_smithi042_260592_object44217:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:04.868074+0000", 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 21.786477528999999, 2024-09-13T14:59:26.723 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21756971, 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 161108 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868074+0000", 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.724 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868074+0000", 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868072+0000", 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.725 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868079+0000", 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5429999999999999e-06 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868090+0000", 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1258e-05 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868096+0000", 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1800000000000001e-06 2024-09-13T14:59:26.726 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868149+0000", 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2729999999999998e-05 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868268+0000", 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011863599999999999 2024-09-13T14:59:26.727 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868703+0000", 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000435516 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.870949+0000", 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022456490000000002 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.728 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.871537+0000", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000588738 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085579+0000", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21404150199999999 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085616+0000", 2024-09-13T14:59:26.729 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6961000000000001e-05 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085643+0000", 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7491000000000002e-05 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162892 12.f 12:f805084e:::benchmark_data_smithi042_260592_object46001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.638024+0000", 2024-09-13T14:59:26.730 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.016527377999999, 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.20950497400000001, 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162892 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638024+0000", 2024-09-13T14:59:26.731 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638024+0000", 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638023+0000", 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.732 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638031+0000", 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5359999999999992e-06 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638040+0000", 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2639999999999997e-06 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638046+0000", 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1309999999999997e-06 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.733 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638089+0000", 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3112000000000003e-05 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638198+0000", 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010893899999999999 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.734 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638308+0000", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110506 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645600+0000", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0072915109999999997 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645605+0000", 2024-09-13T14:59:26.735 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1139999999999997e-06 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847435+0000", 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.201830712 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847479+0000", 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3217999999999999e-05 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.736 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847529+0000", 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0138e-05 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162893 12.f 12:f23c18be:::benchmark_data_smithi042_260592_object46002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.638055+0000", 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.016496592999999, 2024-09-13T14:59:26.737 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.20965494900000001, 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162893 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.738 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638055+0000", 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638055+0000", 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638054+0000", 2024-09-13T14:59:26.739 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638057+0000", 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.644e-06 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638063+0000", 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.144e-06 2024-09-13T14:59:26.740 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.638066+0000", 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2540000000000001e-06 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.641148+0000", 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003082272 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.741 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.641333+0000", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018472200000000001 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.641456+0000", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000122965 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645622+0000", 2024-09-13T14:59:26.742 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041659069999999999 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648172+0000", 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0025498370000000001 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847628+0000", 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19945617800000001 2024-09-13T14:59:26.743 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847674+0000", 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6487999999999999e-05 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847710+0000", 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5049000000000003e-05 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.744 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162902 12.f 12:f9e71d8a:::benchmark_data_smithi042_260592_object46011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.645537+0000", 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.009014354000001, 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.202303599, 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162902 2024-09-13T14:59:26.745 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645537+0000", 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645537+0000", 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.746 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645536+0000", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645544+0000", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7830000000000004e-06 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645556+0000", 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1537e-05 2024-09-13T14:59:26.747 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645563+0000", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1579999999999999e-06 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645641+0000", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7908e-05 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645718+0000", 2024-09-13T14:59:26.748 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7002000000000006e-05 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.645789+0000", 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0790999999999994e-05 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648304+0000", 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0025153850000000002 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.749 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648787+0000", 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048259000000000003 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847763+0000", 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19897589500000001 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847802+0000", 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9690000000000001e-05 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.750 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847840+0000", 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8099000000000002e-05 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162905 12.4 12:23224783:::benchmark_data_smithi042_260592_object46014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.648050+0000", 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.006501188000001, 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19772208599999999, 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.751 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162905 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648050+0000", 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.752 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648050+0000", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648049+0000", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648055+0000", 2024-09-13T14:59:26.753 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3779999999999998e-06 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648065+0000", 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.927e-06 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648070+0000", 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0139999999999998e-06 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.754 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648106+0000", 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6254000000000002e-05 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648190+0000", 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3808999999999997e-05 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:59:26.755 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648614+0000", 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000423256 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.650507+0000", 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001893554 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.756 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.845646+0000", 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19513922 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.845683+0000", 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6665999999999998e-05 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.757 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.845729+0000", 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6063000000000003e-05 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.845772+0000", 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3061e-05 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.758 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162907 12.15 12:ab557e3f:::benchmark_data_smithi042_260592_object46016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.648076+0000", 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.006474967999999, 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19587211600000001, 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.760 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162907 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648076+0000", 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.761 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648076+0000", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648076+0000", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648078+0000", 2024-09-13T14:59:26.762 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7689999999999999e-06 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648081+0000", 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3680000000000002e-06 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648084+0000", 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9730000000000002e-06 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.763 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648121+0000", 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7261999999999997e-05 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648205+0000", 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3727000000000007e-05 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.648613+0000", 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00040823900000000002 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.764 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.650405+0000", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0017916589999999999 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843493+0000", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19308832000000001 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843863+0000", 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00036989100000000001 2024-09-13T14:59:26.765 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843908+0000", 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4953000000000001e-05 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843948+0000", 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0215e-05 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.766 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162909 12.f 12:fd36a84d:::benchmark_data_smithi042_260592_object46018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.651588+0000", 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.002963657999999, 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19641839999999999, 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162909 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.767 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651588+0000", 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651588+0000", 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.768 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651586+0000", 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651597+0000", 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0488e-05 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651609+0000", 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2145e-05 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.769 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651629+0000", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9959999999999999e-05 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.651674+0000", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4934e-05 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843260+0000", 2024-09-13T14:59:26.770 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19158671799999999 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,5", 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843379+0000", 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000118406 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847891+0000", 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045127309999999999 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.771 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847900+0000", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3280000000000006e-06 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847935+0000", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4706999999999998e-05 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847971+0000", 2024-09-13T14:59:26.772 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6378999999999998e-05 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.848006+0000", 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5024999999999998e-05 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45599.0:162892 12.f e324/298 ondisk, result = 0)", 2024-09-13T14:59:26.773 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.652578+0000", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 20.001973194000001, 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.194958355, 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.652578+0000", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.652578+0000", 2024-09-13T14:59:26.774 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.652577+0000", 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.652582+0000", 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1000000000000003e-06 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.775 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842620+0000", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.19003836900000001 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842630+0000", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8919999999999995e-06 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847423+0000", 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047923760000000001 2024-09-13T14:59:26.776 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847430+0000", 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9410000000000001e-06 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847536+0000", 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010665799999999999 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.777 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:162911 12.4 12:271dc227:::benchmark_data_smithi042_260592_object46020:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.655018+0000", 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 19.999533267, 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.199198981, 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 162911 2024-09-13T14:59:26.778 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.655018+0000", 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.655018+0000", 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.779 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.655017+0000", 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.655022+0000", 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6829999999999999e-06 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.780 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842614+0000", 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.187592012 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842631+0000", 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7198e-05 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.781 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.783 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842671+0000", 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0309999999999999e-05 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842803+0000", 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000131784 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.784 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.843298+0000", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00049536000000000003 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.851338+0000", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0080396029999999993 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.851346+0000", 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8110000000000003e-06 2024-09-13T14:59:26.785 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.854099+0000", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027529009999999999 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.854168+0000", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9826999999999994e-05 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.854217+0000", 2024-09-13T14:59:26.786 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8463e-05 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:162910 12.2 e324/307 12:42b085ca:::benchmark_data_smithi042_260592_object46019:head v 324'7309, mlcod=324'7307)", 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:06.656647+0000", 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 19.997904444, 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.190677402, 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:26.787 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.656647+0000", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.656647+0000", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.656646+0000", 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T14:59:26.788 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.656653+0000", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3880000000000002e-06 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842614+0000", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.18596077599999999 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842631+0000", 2024-09-13T14:59:26.789 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6748e-05 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842672+0000", 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1251000000000002e-05 2024-09-13T14:59:26.790 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.842695+0000", 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3156999999999999e-05 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847293+0000", 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045980609999999996 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T14:59:26.791 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:06.847324+0000", 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1084999999999998e-05 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:26.792 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:26.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:26.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T14:59:26.918 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:26.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:26.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:26.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:27.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.070 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:27.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:27.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:27.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:27.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:27.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:27.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:27.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:27.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:27.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T14:59:27.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:27.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:27.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:27.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:27.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:27.876 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:27.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:27.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:27.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:27.933 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:27.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:27 smithi159 ceph-mon[102519]: pgmap v1183: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 28690/167940 objects degraded (17.083%); 6924/167940 objects misplaced (4.123%) 2024-09-13T14:59:28.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:28.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:27 smithi042 ceph-mon[108570]: pgmap v1183: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 28690/167940 objects degraded (17.083%); 6924/167940 objects misplaced (4.123%) 2024-09-13T14:59:28.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:27 smithi042 ceph-mon[113926]: pgmap v1183: 109 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 28690/167940 objects degraded (17.083%); 6924/167940 objects misplaced (4.123%) 2024-09-13T14:59:28.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:28.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:28.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:28.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:28.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:28.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:28.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:28.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:28.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:28.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.735 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:28.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:28.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:28.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:28.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:28.972 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:29.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:29.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:29.106 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:29.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:29.262 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:29.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:29.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:29.398 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:29.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:29.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:29.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.685 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:29.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:29.787 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:29.796 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:29 smithi159 ceph-mon[102519]: pgmap v1184: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27631/161682 objects degraded (17.090%); 6357/161682 objects misplaced (3.932%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:59:29.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:29.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:29.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:29.941 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:29.941 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:183894 12.d e332/298 12:b3f64f8d:::benchmark_data_smithi042_260592_object67003:head v 332'11452, mlcod=332'11450)", 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:29.937948+0000", 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0019902280000000001, 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0020197510000000002, 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937948+0000", 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937948+0000", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937947+0000", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937953+0000", 2024-09-13T14:59:29.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9179999999999999e-06 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937966+0000", 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2653000000000001e-05 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.937971+0000", 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4349999999999996e-06 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.938009+0000", 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7858999999999998e-05 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.938031+0000", 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1792000000000001e-05 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.945 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:183893 12.16 e332/315 12:6ffeaf7b:::benchmark_data_smithi042_260592_object67002:head v 332'8149, mlcod=332'8147)", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:29.938992+0000", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00094616000000000003, 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0010130269999999999, 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.938992+0000", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:29.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.938992+0000", 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.938991+0000", 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939001+0000", 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6600000000000007e-06 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939012+0000", 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1201e-05 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939018+0000", 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1199999999999999e-06 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939058+0000", 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.968e-05 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939082+0000", 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.446e-05 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:183895 12.17 e332/307 12:ea6bfc2d:::benchmark_data_smithi042_260592_object67004:head v 332'8060, mlcod=332'8058)", 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:29.939026+0000", 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00091242600000000004, 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001010356, 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939026+0000", 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939026+0000", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939025+0000", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939029+0000", 2024-09-13T14:59:29.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2279999999999998e-06 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939034+0000", 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.215e-06 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939038+0000", 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7699999999999999e-06 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939078+0000", 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0899000000000003e-05 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:29.939096+0000", 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7197000000000002e-05 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T14:59:29.953 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 3 2024-09-13T14:59:29.954 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:29.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:29.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:30.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:29 smithi042 ceph-mon[108570]: pgmap v1184: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27631/161682 objects degraded (17.090%); 6357/161682 objects misplaced (3.932%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:59:30.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:29 smithi042 ceph-mon[113926]: pgmap v1184: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27631/161682 objects degraded (17.090%); 6357/161682 objects misplaced (3.932%); 6.0 MiB/s, 95 objects/s recovering 2024-09-13T14:59:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:29] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.51.0" 2024-09-13T14:59:30.109 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.233 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.335 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:30.476 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:30.480 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:30.496 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:30.598 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:30.630 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:30.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T14:59:30.732 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:30.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:30.915 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 27631/161682 objects degraded (17.090%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:30.915 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 27631/161682 objects degraded (17.090%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:30.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:30.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:30.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 27631/161682 objects degraded (17.090%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:31.010 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:31.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:31.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:31.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:31.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:31.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:31.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:31.307 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:31.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:31.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:31.487 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:31.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:31.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:31.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:31.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:31.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:31.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:31.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:31.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.004 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:32.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:31 smithi042 ceph-mon[113926]: pgmap v1185: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 27302/158280 objects degraded (17.249%); 6175/158280 objects misplaced (3.901%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:59:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:31 smithi042 ceph-mon[108570]: pgmap v1185: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 27302/158280 objects degraded (17.249%); 6175/158280 objects misplaced (3.901%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:59:32.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:31 smithi159 ceph-mon[102519]: pgmap v1185: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 887 op/s; 27302/158280 objects degraded (17.249%); 6175/158280 objects misplaced (3.901%); 5.4 MiB/s, 86 objects/s recovering 2024-09-13T14:59:32.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:32.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T14:59:32.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:32.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:32.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:32.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:32.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:32.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:32.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:32.945 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:32.977 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:32.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:32.999 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T14:59:33.000 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T14:59:33.000 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T14:59:33.000 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T14:59:33.000 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T14:59:33.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:33.125 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:33.227 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:33.274 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:33.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:33.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:33.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:33.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.551 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:33.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:33.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:33.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:33.803 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:33.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:33.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T14:59:33.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:33.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:33.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:34.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.248 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T14:59:34.249 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:33 smithi042 ceph-mon[108570]: pgmap v1186: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:59:34.249 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:33 smithi042 ceph-mon[113926]: pgmap v1186: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:59:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:33 smithi159 ceph-mon[102519]: pgmap v1186: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 967 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 3.3 MiB/s, 52 objects/s recovering 2024-09-13T14:59:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:34.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:34.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:34.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:34.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:34.680 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:34.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:34.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:34.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:34.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:34.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:34.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:35.043 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:59:35.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:35.085 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:59:35.085 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:35.110 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:35.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:35.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:59:35.159 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:35.206 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:59:35.207 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:59:35.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:35.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:35.241 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:59:35.241 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:35.287 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:59:35.288 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:35.314 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T14:59:35.317 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T14:59:35.334 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:59:35.335 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:35.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:35.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:35.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:35.375 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:59:35.376 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:35.470 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:35.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:35.599 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:35.628 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:35.730 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:35.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:35.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:35.864 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:35.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:35.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:35.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:35.910 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[113926]: pgmap v1187: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1135815748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[113926]: from='client.55318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 25662/146085 objects degraded (17.566%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[108570]: pgmap v1187: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1135815748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[108570]: from='client.55318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:35.911 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 25662/146085 objects degraded (17.566%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:36.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:36.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:36.125 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:35 smithi159 ceph-mon[102519]: pgmap v1187: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 25662/146085 objects degraded (17.566%); 5597/146085 objects misplaced (3.831%); 2.8 MiB/s, 44 objects/s recovering 2024-09-13T14:59:36.125 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1135815748' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:36.125 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:35 smithi159 ceph-mon[102519]: from='client.55318 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T14:59:36.126 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 25662/146085 objects degraded (17.566%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:36.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:36.177 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:36.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:36.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:36.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:36.421 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:36.421 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:36.463 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:36.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:36.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:36.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T14:59:36.704 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T14:59:36.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:36.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:36.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:36.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:36.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:36.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:36.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:37.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:37.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:37.153 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:37.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T14:59:37.219 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:37.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:37.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:37.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:37.478 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:37.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:37.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:37.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:37.679 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:37.780 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:37.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:37.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:37 smithi042 ceph-mon[113926]: pgmap v1188: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 25013/137574 objects degraded (18.181%); 5430/137574 objects misplaced (3.947%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:59:37.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:37 smithi042 ceph-mon[108570]: pgmap v1188: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 25013/137574 objects degraded (18.181%); 5430/137574 objects misplaced (3.947%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:59:37.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:37.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:37.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:38.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:38.101 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1187,"stamp":"2024-09-13T14:59:34.571306+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3190134781,"num_objects":48695,"num_object_clones":0,"num_object_copies":146085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25662,"num_objects_misplaced":5597,"num_objects_unfound":0,"num_objects_dirty":48695,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135899,"num_write_kb":4436482,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6426,"num_bytes_recovered":417274058,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":205383,"ondisk_log_size":205383,"up":288,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18754228,"kb_used_data":11556788,"kb_used_omap":28,"kb_used_meta":7197219,"kb_avail":731206988,"statfs":{"total":767960285184,"available":748755955712,"internally_reserved":0,"allocated":11834150912,"data_stored":11819767584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":7369952657},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,7,6,4,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-634322944,"num_objects":-9679,"num_object_clones":0,"num_object_copies":-29037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3546,"num_objects_misplaced":-1484,"num_objects_unfound":0,"num_objects_dirty":-9679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9679,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":527,"num_bytes_recovered":34537472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.823903"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":489,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752253+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T14:59:22.752253+0000","last_peered":"2024-09-13T14:59:22.752253+0000","last_clean":"2024-09-13T14:59:22.752253+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T14:59:22.752253+0000","last_undegraded":"2024-09-13T14:59:22.752253+0000","last_fullsized":"2024-09-13T14:59:22.752253+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":23,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.765793+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T14:59:21.765793+0000","last_peered":"2024-09-13T14:59:21.765793+0000","last_clean":"2024-09-13T14:59:21.765793+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T14:59:21.765793+0000","last_undegraded":"2024-09-13T14:59:21.765793+0000","last_fullsized":"2024-09-13T14:59:21.765793+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":302,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760690+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:59:21.760690+0000","last_peered":"2024-09-13T14:59:21.760690+0000","last_clean":"2024-09-13T14:59:21.760690+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:59:21.760690+0000","last_undegraded":"2024-09-13T14:59:21.760690+0000","last_fullsized":"2024-09-13T14:59:21.760690+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":299,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.752804+0000","last_change":"2024-09-13T14:58:00.207288+0000","last_active":"2024-09-13T14:59:22.752804+0000","last_peered":"2024-09-13T14:59:22.752804+0000","last_clean":"2024-09-13T14:40:02.364020+0000","last_became_active":"2024-09-13T14:58:00.207288+0000","last_became_peered":"2024-09-13T14:58:00.207288+0000","last_unstale":"2024-09-13T14:59:22.752804+0000","last_undegraded":"2024-09-13T14:59:22.752804+0000","last_fullsized":"2024-09-13T14:58:00.194753+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:56:13.138124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":300,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.752560+0000","last_change":"2024-09-13T14:58:00.207997+0000","last_active":"2024-09-13T14:59:22.752560+0000","last_peered":"2024-09-13T14:59:22.752560+0000","last_clean":"2024-09-13T14:40:02.365015+0000","last_became_active":"2024-09-13T14:58:00.207997+0000","last_became_peered":"2024-09-13T14:58:00.207997+0000","last_unstale":"2024-09-13T14:59:22.752560+0000","last_undegraded":"2024-09-13T14:59:22.752560+0000","last_fullsized":"2024-09-13T14:58:00.194590+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:03.388390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761031+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T14:59:21.761031+0000","last_peered":"2024-09-13T14:59:21.761031+0000","last_clean":"2024-09-13T14:59:21.761031+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T14:59:21.761031+0000","last_undegraded":"2024-09-13T14:59:21.761031+0000","last_fullsized":"2024-09-13T14:59:21.761031+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:05.891150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":514,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752367+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T14:59:22.752367+0000","last_peered":"2024-09-13T14:59:22.752367+0000","last_clean":"2024-09-13T14:59:22.752367+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T14:59:22.752367+0000","last_undegraded":"2024-09-13T14:59:22.752367+0000","last_fullsized":"2024-09-13T14:59:22.752367+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":394,"reported_epoch":331,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:21.756602+0000","last_change":"2024-09-13T14:58:00.194821+0000","last_active":"2024-09-13T14:59:21.756602+0000","last_peered":"2024-09-13T14:59:21.756602+0000","last_clean":"2024-09-13T14:57:53.238105+0000","last_became_active":"2024-09-13T14:58:00.194821+0000","last_became_peered":"2024-09-13T14:58:00.194821+0000","last_unstale":"2024-09-13T14:59:21.756602+0000","last_undegraded":"2024-09-13T14:58:00.191302+0000","last_fullsized":"2024-09-13T14:58:00.191197+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:31:09.722082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"332'7999","reported_seq":9813,"reported_epoch":332,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:32.236021+0000","last_change":"2024-09-13T14:59:13.904786+0000","last_active":"2024-09-13T14:59:32.236021+0000","last_peered":"2024-09-13T14:59:32.236021+0000","last_clean":"2024-09-13T14:59:11.474663+0000","last_became_active":"2024-09-13T14:59:13.846462+0000","last_became_peered":"2024-09-13T14:59:13.846462+0000","last_unstale":"2024-09-13T14:59:32.236021+0000","last_undegraded":"2024-09-13T14:59:13.701425+0000","last_fullsized":"2024-09-13T14:59:13.678382+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":7999,"log_dups_size":0,"ondisk_log_size":7999,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98271232,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":1124,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5681,"num_write_kb":229792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":872,"num_bytes_recovered":57147392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6","objects":369},{"shards":"2,6,7","objects":1124}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":22,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.761867+0000","last_change":"2024-09-13T14:59:13.695251+0000","last_active":"2024-09-13T14:59:21.761867+0000","last_peered":"2024-09-13T14:59:21.761867+0000","last_clean":"2024-09-13T14:59:21.761867+0000","last_became_active":"2024-09-13T14:59:13.687274+0000","last_became_peered":"2024-09-13T14:59:13.687274+0000","last_unstale":"2024-09-13T14:59:21.761867+0000","last_undegraded":"2024-09-13T14:59:21.761867+0000","last_fullsized":"2024-09-13T14:59:21.761867+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:45:08.486245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":288,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.758964+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:59:21.758964+0000","last_peered":"2024-09-13T14:59:21.758964+0000","last_clean":"2024-09-13T14:59:21.758964+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:59:21.758964+0000","last_undegraded":"2024-09-13T14:59:21.758964+0000","last_fullsized":"2024-09-13T14:59:21.758964+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"332'8281","reported_seq":8338,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.750785+0000","last_change":"2024-09-13T14:58:32.581659+0000","last_active":"2024-09-13T14:59:30.750785+0000","last_peered":"2024-09-13T14:59:30.750785+0000","last_clean":"2024-09-13T14:59:30.750785+0000","last_became_active":"2024-09-13T14:57:42.090080+0000","last_became_peered":"2024-09-13T14:57:42.090080+0000","last_unstale":"2024-09-13T14:59:30.750785+0000","last_undegraded":"2024-09-13T14:59:30.750785+0000","last_fullsized":"2024-09-13T14:59:30.750785+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":8281,"log_dups_size":0,"ondisk_log_size":8281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99188736,"num_objects":1513,"num_object_clones":0,"num_object_copies":4539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5833,"num_write_kb":235104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":21,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.762303+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T14:59:21.762303+0000","last_peered":"2024-09-13T14:59:21.762303+0000","last_clean":"2024-09-13T14:59:21.762303+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T14:59:21.762303+0000","last_undegraded":"2024-09-13T14:59:21.762303+0000","last_fullsized":"2024-09-13T14:59:21.762303+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:49.409549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761219+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T14:59:21.761219+0000","last_peered":"2024-09-13T14:59:21.761219+0000","last_clean":"2024-09-13T14:59:21.761219+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T14:59:21.761219+0000","last_undegraded":"2024-09-13T14:59:21.761219+0000","last_fullsized":"2024-09-13T14:59:21.761219+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"332'8037","reported_seq":8167,"reported_epoch":332,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:31.852566+0000","last_change":"2024-09-13T14:59:13.904720+0000","last_active":"2024-09-13T14:59:31.852566+0000","last_peered":"2024-09-13T14:59:31.852566+0000","last_clean":"2024-09-13T14:59:11.647182+0000","last_became_active":"2024-09-13T14:59:13.880217+0000","last_became_peered":"2024-09-13T14:59:13.880217+0000","last_unstale":"2024-09-13T14:59:31.852566+0000","last_undegraded":"2024-09-13T14:59:13.709724+0000","last_fullsized":"2024-09-13T14:59:13.682580+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'4893","last_scrub_stamp":"2024-09-13T14:57:49.600786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.600786+0000","objects_scrubbed":2644,"log_size":8037,"log_dups_size":0,"ondisk_log_size":8037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":823,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96600064,"num_objects":1474,"num_object_clones":0,"num_object_copies":4422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3152,"num_write_kb":63424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":1474},{"shards":"3,7","objects":-567}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":206,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760542+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:59:21.760542+0000","last_peered":"2024-09-13T14:59:21.760542+0000","last_clean":"2024-09-13T14:59:21.760542+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:59:21.760542+0000","last_undegraded":"2024-09-13T14:59:21.760542+0000","last_fullsized":"2024-09-13T14:59:21.760542+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":304,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.756592+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:59:21.756592+0000","last_peered":"2024-09-13T14:59:21.756592+0000","last_clean":"2024-09-13T14:59:21.756592+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:59:21.756592+0000","last_undegraded":"2024-09-13T14:59:21.756592+0000","last_fullsized":"2024-09-13T14:59:21.756592+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"332'8337","reported_seq":8468,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:31.865844+0000","last_change":"2024-09-13T14:58:00.208715+0000","last_active":"2024-09-13T14:59:31.865844+0000","last_peered":"2024-09-13T14:59:31.865844+0000","last_clean":"2024-09-13T14:57:59.112552+0000","last_became_active":"2024-09-13T14:58:00.208715+0000","last_became_peered":"2024-09-13T14:58:00.208715+0000","last_unstale":"2024-09-13T14:59:31.865844+0000","last_undegraded":"2024-09-13T14:58:00.197342+0000","last_fullsized":"2024-09-13T14:58:00.197242+0000","mapping_epoch":315,"log_start":"301'211","ondisk_log_start":"301'211","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5072","last_scrub_stamp":"2024-09-13T14:57:49.601547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.601547+0000","objects_scrubbed":2726,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:01.037695+0000","scrub_duration":891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3278,"num_write_kb":65600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1498}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":304,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760619+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:59:21.760619+0000","last_peered":"2024-09-13T14:59:21.760619+0000","last_clean":"2024-09-13T14:59:21.760619+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:59:21.760619+0000","last_undegraded":"2024-09-13T14:59:21.760619+0000","last_fullsized":"2024-09-13T14:59:21.760619+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":332,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:22.754389+0000","last_change":"2024-09-13T14:59:13.689224+0000","last_active":"2024-09-13T14:59:22.754389+0000","last_peered":"2024-09-13T14:59:22.754389+0000","last_clean":"2024-09-13T14:59:22.754389+0000","last_became_active":"2024-09-13T14:59:13.688798+0000","last_became_peered":"2024-09-13T14:59:13.688798+0000","last_unstale":"2024-09-13T14:59:22.754389+0000","last_undegraded":"2024-09-13T14:59:22.754389+0000","last_fullsized":"2024-09-13T14:59:22.754389+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:54:51.804710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.4","version":"332'8126","reported_seq":8248,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.879041+0000","last_change":"2024-09-13T14:57:55.202026+0000","last_active":"2024-09-13T14:59:30.879041+0000","last_peered":"2024-09-13T14:59:30.879041+0000","last_clean":"2024-09-13T14:59:30.879041+0000","last_became_active":"2024-09-13T14:57:42.089977+0000","last_became_peered":"2024-09-13T14:57:42.089977+0000","last_unstale":"2024-09-13T14:59:30.879041+0000","last_undegraded":"2024-09-13T14:59:30.879041+0000","last_fullsized":"2024-09-13T14:59:30.879041+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"314'5167","last_scrub_stamp":"2024-09-13T14:57:55.201860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:55.201860+0000","objects_scrubbed":2796,"log_size":8126,"log_dups_size":0,"ondisk_log_size":8126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:43.151657+0000","scrub_duration":0.69252925899999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103284736,"num_objects":1576,"num_object_clones":0,"num_object_copies":4728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2970,"num_write_kb":56000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":839,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.751755+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T14:59:22.751755+0000","last_peered":"2024-09-13T14:59:22.751755+0000","last_clean":"2024-09-13T14:59:22.751755+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T14:59:22.751755+0000","last_undegraded":"2024-09-13T14:59:22.751755+0000","last_fullsized":"2024-09-13T14:59:22.751755+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:47.007980+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":546,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:22.753076+0000","last_change":"2024-09-13T14:59:13.698027+0000","last_active":"2024-09-13T14:59:22.753076+0000","last_peered":"2024-09-13T14:59:22.753076+0000","last_clean":"2024-09-13T14:59:22.753076+0000","last_became_active":"2024-09-13T14:59:13.689548+0000","last_became_peered":"2024-09-13T14:59:13.689548+0000","last_unstale":"2024-09-13T14:59:22.753076+0000","last_undegraded":"2024-09-13T14:59:22.753076+0000","last_fullsized":"2024-09-13T14:59:22.753076+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:52.358606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761176+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T14:59:21.761176+0000","last_peered":"2024-09-13T14:59:21.761176+0000","last_clean":"2024-09-13T14:59:21.761176+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T14:59:21.761176+0000","last_undegraded":"2024-09-13T14:59:21.761176+0000","last_fullsized":"2024-09-13T14:59:21.761176+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"332'8131","reported_seq":8251,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.781139+0000","last_change":"2024-09-13T14:57:54.124864+0000","last_active":"2024-09-13T14:59:30.781139+0000","last_peered":"2024-09-13T14:59:30.781139+0000","last_clean":"2024-09-13T14:59:30.781139+0000","last_became_active":"2024-09-13T14:57:42.086633+0000","last_became_peered":"2024-09-13T14:57:42.086633+0000","last_unstale":"2024-09-13T14:59:30.781139+0000","last_undegraded":"2024-09-13T14:59:30.781139+0000","last_fullsized":"2024-09-13T14:59:30.781139+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"314'5087","last_scrub_stamp":"2024-09-13T14:57:54.124643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:54.124643+0000","objects_scrubbed":2709,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:22.438092+0000","scrub_duration":0.60692724600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97124352,"num_objects":1482,"num_object_clones":0,"num_object_copies":4446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3049,"num_write_kb":58304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":573,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.758755+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:59:21.758755+0000","last_peered":"2024-09-13T14:59:21.758755+0000","last_clean":"2024-09-13T14:59:21.758755+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:59:21.758755+0000","last_undegraded":"2024-09-13T14:59:21.758755+0000","last_fullsized":"2024-09-13T14:59:21.758755+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":352,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.756718+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T14:59:21.756718+0000","last_peered":"2024-09-13T14:59:21.756718+0000","last_clean":"2024-09-13T14:59:21.756718+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T14:59:21.756718+0000","last_undegraded":"2024-09-13T14:59:21.756718+0000","last_fullsized":"2024-09-13T14:59:21.756718+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761363+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T14:59:21.761363+0000","last_peered":"2024-09-13T14:59:21.761363+0000","last_clean":"2024-09-13T14:59:21.761363+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T14:59:21.761363+0000","last_undegraded":"2024-09-13T14:59:21.761363+0000","last_fullsized":"2024-09-13T14:59:21.761363+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"332'7966","reported_seq":8095,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:29.672275+0000","last_change":"2024-09-13T14:58:00.209329+0000","last_active":"2024-09-13T14:59:29.672275+0000","last_peered":"2024-09-13T14:59:29.672275+0000","last_clean":"2024-09-13T14:57:59.167005+0000","last_became_active":"2024-09-13T14:58:00.209329+0000","last_became_peered":"2024-09-13T14:58:00.209329+0000","last_unstale":"2024-09-13T14:59:29.672275+0000","last_undegraded":"2024-09-13T14:58:00.192021+0000","last_fullsized":"2024-09-13T14:58:00.191919+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5005","last_scrub_stamp":"2024-09-13T14:57:50.368944+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.368944+0000","objects_scrubbed":2603,"log_size":7966,"log_dups_size":0,"ondisk_log_size":7966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:17.407794+0000","scrub_duration":369,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2961,"num_write_kb":59392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1498}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761102+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T14:59:21.761102+0000","last_peered":"2024-09-13T14:59:21.761102+0000","last_clean":"2024-09-13T14:59:21.761102+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T14:59:21.761102+0000","last_undegraded":"2024-09-13T14:59:21.761102+0000","last_fullsized":"2024-09-13T14:59:21.761102+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.766082+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T14:59:21.766082+0000","last_peered":"2024-09-13T14:59:21.766082+0000","last_clean":"2024-09-13T14:59:21.766082+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T14:59:21.766082+0000","last_undegraded":"2024-09-13T14:59:21.766082+0000","last_fullsized":"2024-09-13T14:59:21.766082+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"332'8055","reported_seq":8180,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:29.655204+0000","last_change":"2024-09-13T14:57:49.747386+0000","last_active":"2024-09-13T14:59:29.655204+0000","last_peered":"2024-09-13T14:59:29.655204+0000","last_clean":"2024-09-13T14:59:29.655204+0000","last_became_active":"2024-09-13T14:57:42.095484+0000","last_became_peered":"2024-09-13T14:57:42.095484+0000","last_unstale":"2024-09-13T14:59:29.655204+0000","last_undegraded":"2024-09-13T14:59:29.655204+0000","last_fullsized":"2024-09-13T14:59:29.655204+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5022","last_scrub_stamp":"2024-09-13T14:57:49.747144+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.747144+0000","objects_scrubbed":2617,"log_size":8055,"log_dups_size":0,"ondisk_log_size":8055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:01.334229+0000","scrub_duration":717,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3042,"num_write_kb":63552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":573,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.759659+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:59:21.759659+0000","last_peered":"2024-09-13T14:59:21.759659+0000","last_clean":"2024-09-13T14:59:21.759659+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:59:21.759659+0000","last_undegraded":"2024-09-13T14:59:21.759659+0000","last_fullsized":"2024-09-13T14:59:21.759659+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":491,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752622+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T14:59:22.752622+0000","last_peered":"2024-09-13T14:59:22.752622+0000","last_clean":"2024-09-13T14:59:22.752622+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T14:59:22.752622+0000","last_undegraded":"2024-09-13T14:59:22.752622+0000","last_fullsized":"2024-09-13T14:59:22.752622+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":504,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.755242+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T14:59:22.755242+0000","last_peered":"2024-09-13T14:59:22.755242+0000","last_clean":"2024-09-13T14:59:22.755242+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T14:59:22.755242+0000","last_undegraded":"2024-09-13T14:59:22.755242+0000","last_fullsized":"2024-09-13T14:59:22.755242+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"332'11436","reported_seq":11522,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:29.674716+0000","last_change":"2024-09-13T14:56:44.021769+0000","last_active":"2024-09-13T14:59:29.674716+0000","last_peered":"2024-09-13T14:59:29.674716+0000","last_clean":"2024-09-13T14:59:29.674716+0000","last_became_active":"2024-09-13T14:56:44.021451+0000","last_became_peered":"2024-09-13T14:56:44.021451+0000","last_unstale":"2024-09-13T14:59:29.674716+0000","last_undegraded":"2024-09-13T14:59:29.674716+0000","last_fullsized":"2024-09-13T14:59:29.674716+0000","mapping_epoch":298,"log_start":"308'5100","ondisk_log_start":"308'5100","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6336,"log_dups_size":0,"ondisk_log_size":6336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:41:29.177584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205062168,"num_objects":3130,"num_object_clones":0,"num_object_copies":9390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3130,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11436,"num_write_kb":466049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":537,"reported_epoch":331,"state":"active+undersized","last_fresh":"2024-09-13T14:59:21.756756+0000","last_change":"2024-09-13T14:58:00.196065+0000","last_active":"2024-09-13T14:59:21.756756+0000","last_peered":"2024-09-13T14:59:21.756756+0000","last_clean":"2024-09-13T14:54:54.194954+0000","last_became_active":"2024-09-13T14:58:00.196065+0000","last_became_peered":"2024-09-13T14:58:00.196065+0000","last_unstale":"2024-09-13T14:59:21.756756+0000","last_undegraded":"2024-09-13T14:59:21.756756+0000","last_fullsized":"2024-09-13T14:58:00.191925+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:22:33.310023+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":319,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760191+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:59:21.760191+0000","last_peered":"2024-09-13T14:59:21.760191+0000","last_clean":"2024-09-13T14:59:21.760191+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:59:21.760191+0000","last_undegraded":"2024-09-13T14:59:21.760191+0000","last_fullsized":"2024-09-13T14:59:21.760191+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":477,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752077+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T14:59:22.752077+0000","last_peered":"2024-09-13T14:59:22.752077+0000","last_clean":"2024-09-13T14:59:22.752077+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T14:59:22.752077+0000","last_undegraded":"2024-09-13T14:59:22.752077+0000","last_fullsized":"2024-09-13T14:59:22.752077+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":314,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.756964+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T14:59:21.756964+0000","last_peered":"2024-09-13T14:59:21.756964+0000","last_clean":"2024-09-13T14:59:21.756964+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T14:59:21.756964+0000","last_undegraded":"2024-09-13T14:59:21.756964+0000","last_fullsized":"2024-09-13T14:59:21.756964+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:13.033398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"332'11594","reported_seq":11683,"reported_epoch":332,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T14:59:31.943938+0000","last_change":"2024-09-13T14:59:13.842972+0000","last_active":"2024-09-13T14:59:31.943938+0000","last_peered":"2024-09-13T14:59:31.943938+0000","last_clean":"2024-09-13T14:57:59.154960+0000","last_became_active":"2024-09-13T14:59:13.815667+0000","last_became_peered":"2024-09-13T14:59:13.815667+0000","last_unstale":"2024-09-13T14:59:31.943938+0000","last_undegraded":"2024-09-13T14:59:13.701823+0000","last_fullsized":"2024-09-13T14:59:13.684281+0000","mapping_epoch":326,"log_start":"314'5671","ondisk_log_start":"314'5671","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":5923,"log_dups_size":0,"ondisk_log_size":5923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190709760,"num_objects":2910,"num_object_clones":0,"num_object_copies":8730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2910,"num_objects_misplaced":2910,"num_objects_unfound":0,"num_objects_dirty":2910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11594,"num_write_kb":464128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":2910}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":770,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754289+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T14:59:22.754289+0000","last_peered":"2024-09-13T14:59:22.754289+0000","last_clean":"2024-09-13T14:59:22.754289+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T14:59:22.754289+0000","last_undegraded":"2024-09-13T14:59:22.754289+0000","last_fullsized":"2024-09-13T14:59:22.754289+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:46.891038+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":481,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.753735+0000","last_change":"2024-09-13T14:58:00.196079+0000","last_active":"2024-09-13T14:59:22.753735+0000","last_peered":"2024-09-13T14:59:22.753735+0000","last_clean":"2024-09-13T14:57:53.147377+0000","last_became_active":"2024-09-13T14:58:00.196079+0000","last_became_peered":"2024-09-13T14:58:00.196079+0000","last_unstale":"2024-09-13T14:59:22.753735+0000","last_undegraded":"2024-09-13T14:59:22.753735+0000","last_fullsized":"2024-09-13T14:58:00.192042+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:50.213407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":18,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.760837+0000","last_change":"2024-09-13T14:59:13.720055+0000","last_active":"2024-09-13T14:59:21.760837+0000","last_peered":"2024-09-13T14:59:21.760837+0000","last_clean":"2024-09-13T14:59:21.760837+0000","last_became_active":"2024-09-13T14:59:13.719639+0000","last_became_peered":"2024-09-13T14:59:13.719639+0000","last_unstale":"2024-09-13T14:59:21.760837+0000","last_undegraded":"2024-09-13T14:59:21.760837+0000","last_fullsized":"2024-09-13T14:59:21.760837+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:48.915754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,5],"avail_no_missing":["4","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":371,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.754921+0000","last_change":"2024-09-13T14:58:00.197201+0000","last_active":"2024-09-13T14:59:22.754921+0000","last_peered":"2024-09-13T14:59:22.754921+0000","last_clean":"2024-09-13T14:54:54.205156+0000","last_became_active":"2024-09-13T14:58:00.197201+0000","last_became_peered":"2024-09-13T14:58:00.197201+0000","last_unstale":"2024-09-13T14:59:22.754921+0000","last_undegraded":"2024-09-13T14:59:22.754921+0000","last_fullsized":"2024-09-13T14:58:00.190432+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:48.403571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"332'11535","reported_seq":11579,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.861660+0000","last_change":"2024-09-13T14:56:43.814995+0000","last_active":"2024-09-13T14:59:30.861660+0000","last_peered":"2024-09-13T14:59:30.861660+0000","last_clean":"2024-09-13T14:59:30.861660+0000","last_became_active":"2024-09-13T14:56:43.814719+0000","last_became_peered":"2024-09-13T14:56:43.814719+0000","last_unstale":"2024-09-13T14:59:30.861660+0000","last_undegraded":"2024-09-13T14:59:30.861660+0000","last_fullsized":"2024-09-13T14:59:30.861660+0000","mapping_epoch":298,"log_start":"308'5000","ondisk_log_start":"308'5000","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.360102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197066752,"num_objects":3007,"num_object_clones":0,"num_object_copies":9021,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3007,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11535,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":584,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.761704+0000","last_change":"2024-09-13T14:59:15.701141+0000","last_active":"2024-09-13T14:59:21.761704+0000","last_peered":"2024-09-13T14:59:21.761704+0000","last_clean":"2024-09-13T14:59:21.761704+0000","last_became_active":"2024-09-13T14:59:15.700696+0000","last_became_peered":"2024-09-13T14:59:15.700696+0000","last_unstale":"2024-09-13T14:59:21.761704+0000","last_undegraded":"2024-09-13T14:59:21.761704+0000","last_fullsized":"2024-09-13T14:59:21.761704+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:01:36.489123+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":533,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.749094+0000","last_change":"2024-09-13T14:58:00.209707+0000","last_active":"2024-09-13T14:59:22.749094+0000","last_peered":"2024-09-13T14:59:22.749094+0000","last_clean":"2024-09-13T14:57:53.147953+0000","last_became_active":"2024-09-13T14:58:00.209707+0000","last_became_peered":"2024-09-13T14:58:00.209707+0000","last_unstale":"2024-09-13T14:59:22.749094+0000","last_undegraded":"2024-09-13T14:59:22.749094+0000","last_fullsized":"2024-09-13T14:58:00.188724+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:44.693424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":489,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754081+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T14:59:22.754081+0000","last_peered":"2024-09-13T14:59:22.754081+0000","last_clean":"2024-09-13T14:59:22.754081+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T14:59:22.754081+0000","last_undegraded":"2024-09-13T14:59:22.754081+0000","last_fullsized":"2024-09-13T14:59:22.754081+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":543,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.754981+0000","last_change":"2024-09-13T14:58:00.196078+0000","last_active":"2024-09-13T14:59:22.754981+0000","last_peered":"2024-09-13T14:59:22.754981+0000","last_clean":"2024-09-13T14:57:53.149012+0000","last_became_active":"2024-09-13T14:58:00.196078+0000","last_became_peered":"2024-09-13T14:58:00.196078+0000","last_unstale":"2024-09-13T14:59:22.754981+0000","last_undegraded":"2024-09-13T14:59:22.754981+0000","last_fullsized":"2024-09-13T14:58:00.192052+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:57.351796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"332'11310","reported_seq":11378,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:30.903576+0000","last_change":"2024-09-13T14:58:00.200526+0000","last_active":"2024-09-13T14:59:30.903576+0000","last_peered":"2024-09-13T14:59:30.903576+0000","last_clean":"2024-09-13T14:57:59.094224+0000","last_became_active":"2024-09-13T14:58:00.200526+0000","last_became_peered":"2024-09-13T14:58:00.200526+0000","last_unstale":"2024-09-13T14:59:30.903576+0000","last_undegraded":"2024-09-13T14:58:00.190108+0000","last_fullsized":"2024-09-13T14:58:00.189997+0000","mapping_epoch":315,"log_start":"302'2200","ondisk_log_start":"302'2200","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":9110,"log_dups_size":0,"ondisk_log_size":9110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:50.813569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201719808,"num_objects":3078,"num_object_clones":0,"num_object_copies":9234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3078,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11310,"num_write_kb":460416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":3078}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":613,"reported_epoch":331,"state":"active+undersized","last_fresh":"2024-09-13T14:59:21.762053+0000","last_change":"2024-09-13T14:58:00.206124+0000","last_active":"2024-09-13T14:59:21.762053+0000","last_peered":"2024-09-13T14:59:21.762053+0000","last_clean":"2024-09-13T14:57:53.396018+0000","last_became_active":"2024-09-13T14:58:00.206124+0000","last_became_peered":"2024-09-13T14:58:00.206124+0000","last_unstale":"2024-09-13T14:59:21.762053+0000","last_undegraded":"2024-09-13T14:59:21.762053+0000","last_fullsized":"2024-09-13T14:58:00.189988+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:24.008447+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":320,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.759948+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:59:21.759948+0000","last_peered":"2024-09-13T14:59:21.759948+0000","last_clean":"2024-09-13T14:59:21.759948+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:59:21.759948+0000","last_undegraded":"2024-09-13T14:59:21.759948+0000","last_fullsized":"2024-09-13T14:59:21.759948+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":18,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.765607+0000","last_change":"2024-09-13T14:59:13.687964+0000","last_active":"2024-09-13T14:59:21.765607+0000","last_peered":"2024-09-13T14:59:21.765607+0000","last_clean":"2024-09-13T14:59:21.765607+0000","last_became_active":"2024-09-13T14:59:13.687546+0000","last_became_peered":"2024-09-13T14:59:13.687546+0000","last_unstale":"2024-09-13T14:59:21.765607+0000","last_undegraded":"2024-09-13T14:59:21.765607+0000","last_fullsized":"2024-09-13T14:59:21.765607+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:22.830176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":502,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754070+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T14:59:22.754070+0000","last_peered":"2024-09-13T14:59:22.754070+0000","last_clean":"2024-09-13T14:59:22.754070+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T14:59:22.754070+0000","last_undegraded":"2024-09-13T14:59:22.754070+0000","last_fullsized":"2024-09-13T14:59:22.754070+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"332'11647","reported_seq":11733,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:32.214790+0000","last_change":"2024-09-13T14:58:00.209181+0000","last_active":"2024-09-13T14:59:32.214790+0000","last_peered":"2024-09-13T14:59:32.214790+0000","last_clean":"2024-09-13T14:57:59.161016+0000","last_became_active":"2024-09-13T14:58:00.209181+0000","last_became_peered":"2024-09-13T14:58:00.209181+0000","last_unstale":"2024-09-13T14:59:32.214790+0000","last_undegraded":"2024-09-13T14:58:00.188435+0000","last_fullsized":"2024-09-13T14:58:00.188304+0000","mapping_epoch":315,"log_start":"308'4800","ondisk_log_start":"308'4800","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:50.670496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197591040,"num_objects":3015,"num_object_clones":0,"num_object_copies":9045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3015,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11647,"num_write_kb":469184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":3015}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":581,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.756678+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:59:21.756678+0000","last_peered":"2024-09-13T14:59:21.756678+0000","last_clean":"2024-09-13T14:59:21.756678+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:59:21.756678+0000","last_undegraded":"2024-09-13T14:59:21.756678+0000","last_fullsized":"2024-09-13T14:59:21.756678+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":320,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760880+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:59:21.760880+0000","last_peered":"2024-09-13T14:59:21.760880+0000","last_clean":"2024-09-13T14:59:21.760880+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:59:21.760880+0000","last_undegraded":"2024-09-13T14:59:21.760880+0000","last_fullsized":"2024-09-13T14:59:21.760880+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":558,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.754345+0000","last_change":"2024-09-13T14:58:00.198485+0000","last_active":"2024-09-13T14:59:22.754345+0000","last_peered":"2024-09-13T14:59:22.754345+0000","last_clean":"2024-09-13T14:57:53.148529+0000","last_became_active":"2024-09-13T14:58:00.198485+0000","last_became_peered":"2024-09-13T14:58:00.198485+0000","last_unstale":"2024-09-13T14:59:22.754345+0000","last_undegraded":"2024-09-13T14:59:22.754345+0000","last_fullsized":"2024-09-13T14:58:00.194105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:25.937883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":428,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.753957+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T14:59:22.753957+0000","last_peered":"2024-09-13T14:59:22.753957+0000","last_clean":"2024-09-13T14:59:22.753957+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T14:59:22.753957+0000","last_undegraded":"2024-09-13T14:59:22.753957+0000","last_fullsized":"2024-09-13T14:59:22.753957+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"332'11657","reported_seq":11742,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:31.928100+0000","last_change":"2024-09-13T14:56:43.815601+0000","last_active":"2024-09-13T14:59:31.928100+0000","last_peered":"2024-09-13T14:59:31.928100+0000","last_clean":"2024-09-13T14:59:31.928100+0000","last_became_active":"2024-09-13T14:56:43.815322+0000","last_became_peered":"2024-09-13T14:56:43.815322+0000","last_unstale":"2024-09-13T14:59:31.928100+0000","last_undegraded":"2024-09-13T14:59:31.928100+0000","last_fullsized":"2024-09-13T14:59:31.928100+0000","mapping_epoch":298,"log_start":"314'5700","ondisk_log_start":"314'5700","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:30:49.708244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192872448,"num_objects":2943,"num_object_clones":0,"num_object_copies":8829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11657,"num_write_kb":467200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":621,"reported_epoch":331,"state":"active+undersized","last_fresh":"2024-09-13T14:59:21.756923+0000","last_change":"2024-09-13T14:58:00.211752+0000","last_active":"2024-09-13T14:59:21.756923+0000","last_peered":"2024-09-13T14:59:21.756923+0000","last_clean":"2024-09-13T14:57:53.239177+0000","last_became_active":"2024-09-13T14:58:00.211752+0000","last_became_peered":"2024-09-13T14:58:00.211752+0000","last_unstale":"2024-09-13T14:59:21.756923+0000","last_undegraded":"2024-09-13T14:59:21.756923+0000","last_fullsized":"2024-09-13T14:58:00.191604+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:25.585853+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":489,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754166+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T14:59:22.754166+0000","last_peered":"2024-09-13T14:59:22.754166+0000","last_clean":"2024-09-13T14:59:22.754166+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T14:59:22.754166+0000","last_undegraded":"2024-09-13T14:59:22.754166+0000","last_fullsized":"2024-09-13T14:59:22.754166+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.761582+0000","last_change":"2024-09-13T14:59:13.687794+0000","last_active":"2024-09-13T14:59:21.761582+0000","last_peered":"2024-09-13T14:59:21.761582+0000","last_clean":"2024-09-13T14:59:21.761582+0000","last_became_active":"2024-09-13T14:59:13.687365+0000","last_became_peered":"2024-09-13T14:59:13.687365+0000","last_unstale":"2024-09-13T14:59:21.761582+0000","last_undegraded":"2024-09-13T14:59:21.761582+0000","last_fullsized":"2024-09-13T14:59:21.761582+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:34.937515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":123,"reported_epoch":331,"state":"active+undersized","last_fresh":"2024-09-13T14:59:21.760063+0000","last_change":"2024-09-13T14:58:00.196530+0000","last_active":"2024-09-13T14:59:21.760063+0000","last_peered":"2024-09-13T14:59:21.760063+0000","last_clean":"2024-09-13T14:20:45.753839+0000","last_became_active":"2024-09-13T14:58:00.196530+0000","last_became_peered":"2024-09-13T14:58:00.196530+0000","last_unstale":"2024-09-13T14:59:21.760063+0000","last_undegraded":"2024-09-13T14:59:21.760063+0000","last_fullsized":"2024-09-13T14:58:00.192715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:08:52.804801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"332'8249","reported_seq":8381,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:31.918355+0000","last_change":"2024-09-13T14:58:00.267572+0000","last_active":"2024-09-13T14:59:31.918355+0000","last_peered":"2024-09-13T14:59:31.918355+0000","last_clean":"2024-09-13T14:57:59.131526+0000","last_became_active":"2024-09-13T14:58:00.267572+0000","last_became_peered":"2024-09-13T14:58:00.267572+0000","last_unstale":"2024-09-13T14:59:31.918355+0000","last_undegraded":"2024-09-13T14:58:00.194520+0000","last_fullsized":"2024-09-13T14:58:00.194409+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5084","last_scrub_stamp":"2024-09-13T14:57:50.981843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.981843+0000","objects_scrubbed":2677,"log_size":8249,"log_dups_size":0,"ondisk_log_size":8249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:11.598922+0000","scrub_duration":264,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98828288,"num_objects":1508,"num_object_clones":0,"num_object_copies":4524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1508,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3165,"num_write_kb":63872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1508}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":510,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761497+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T14:59:21.761497+0000","last_peered":"2024-09-13T14:59:21.761497+0000","last_clean":"2024-09-13T14:59:21.761497+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T14:59:21.761497+0000","last_undegraded":"2024-09-13T14:59:21.761497+0000","last_fullsized":"2024-09-13T14:59:21.761497+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":503,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:22.752584+0000","last_change":"2024-09-13T14:58:00.206311+0000","last_active":"2024-09-13T14:59:22.752584+0000","last_peered":"2024-09-13T14:59:22.752584+0000","last_clean":"2024-09-13T14:57:56.555019+0000","last_became_active":"2024-09-13T14:58:00.206311+0000","last_became_peered":"2024-09-13T14:58:00.206311+0000","last_unstale":"2024-09-13T14:59:22.752584+0000","last_undegraded":"2024-09-13T14:58:00.194308+0000","last_fullsized":"2024-09-13T14:58:00.193842+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:05.897239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":503,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.755245+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T14:59:22.755245+0000","last_peered":"2024-09-13T14:59:22.755245+0000","last_clean":"2024-09-13T14:59:22.755245+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T14:59:22.755245+0000","last_undegraded":"2024-09-13T14:59:22.755245+0000","last_fullsized":"2024-09-13T14:59:22.755245+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":507,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.751914+0000","last_change":"2024-09-13T14:58:00.200367+0000","last_active":"2024-09-13T14:59:22.751914+0000","last_peered":"2024-09-13T14:59:22.751914+0000","last_clean":"2024-09-13T14:57:53.144224+0000","last_became_active":"2024-09-13T14:58:00.200367+0000","last_became_peered":"2024-09-13T14:58:00.200367+0000","last_unstale":"2024-09-13T14:59:22.751914+0000","last_undegraded":"2024-09-13T14:59:22.751914+0000","last_fullsized":"2024-09-13T14:58:00.192544+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:08.459840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"332'8302","reported_seq":8434,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:32.226975+0000","last_change":"2024-09-13T14:57:50.265028+0000","last_active":"2024-09-13T14:59:32.226975+0000","last_peered":"2024-09-13T14:59:32.226975+0000","last_clean":"2024-09-13T14:59:32.226975+0000","last_became_active":"2024-09-13T14:57:42.096368+0000","last_became_peered":"2024-09-13T14:57:42.096368+0000","last_unstale":"2024-09-13T14:59:32.226975+0000","last_undegraded":"2024-09-13T14:59:32.226975+0000","last_fullsized":"2024-09-13T14:59:32.226975+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5129","last_scrub_stamp":"2024-09-13T14:57:50.264920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.264920+0000","objects_scrubbed":2737,"log_size":8302,"log_dups_size":0,"ondisk_log_size":8302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:16.799883+0000","scrub_duration":521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98959360,"num_objects":1510,"num_object_clones":0,"num_object_copies":4530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3173,"num_write_kb":62272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":749,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752047+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T14:59:22.752047+0000","last_peered":"2024-09-13T14:59:22.752047+0000","last_clean":"2024-09-13T14:59:22.752047+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T14:59:22.752047+0000","last_undegraded":"2024-09-13T14:59:22.752047+0000","last_fullsized":"2024-09-13T14:59:22.752047+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:53.472897+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":319,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:22.755058+0000","last_change":"2024-09-13T14:59:13.691543+0000","last_active":"2024-09-13T14:59:22.755058+0000","last_peered":"2024-09-13T14:59:22.755058+0000","last_clean":"2024-09-13T14:59:22.755058+0000","last_became_active":"2024-09-13T14:59:13.687162+0000","last_became_peered":"2024-09-13T14:59:13.687162+0000","last_unstale":"2024-09-13T14:59:22.755058+0000","last_undegraded":"2024-09-13T14:59:22.755058+0000","last_fullsized":"2024-09-13T14:59:22.755058+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:32.615151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":318,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.757300+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:59:21.757300+0000","last_peered":"2024-09-13T14:59:21.757300+0000","last_clean":"2024-09-13T14:59:21.757300+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:59:21.757300+0000","last_undegraded":"2024-09-13T14:59:21.757300+0000","last_fullsized":"2024-09-13T14:59:21.757300+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760959+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T14:59:21.760959+0000","last_peered":"2024-09-13T14:59:21.760959+0000","last_clean":"2024-09-13T14:59:21.760959+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T14:59:21.760959+0000","last_undegraded":"2024-09-13T14:59:21.760959+0000","last_fullsized":"2024-09-13T14:59:21.760959+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:41.734273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.765722+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T14:59:21.765722+0000","last_peered":"2024-09-13T14:59:21.765722+0000","last_clean":"2024-09-13T14:59:21.765722+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T14:59:21.765722+0000","last_undegraded":"2024-09-13T14:59:21.765722+0000","last_fullsized":"2024-09-13T14:59:21.765722+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":27,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761099+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T14:59:21.761099+0000","last_peered":"2024-09-13T14:59:21.761099+0000","last_clean":"2024-09-13T14:59:21.761099+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T14:59:21.761099+0000","last_undegraded":"2024-09-13T14:59:21.761099+0000","last_fullsized":"2024-09-13T14:59:21.761099+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":409,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754186+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T14:59:22.754186+0000","last_peered":"2024-09-13T14:59:22.754186+0000","last_clean":"2024-09-13T14:59:22.754186+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T14:59:22.754186+0000","last_undegraded":"2024-09-13T14:59:22.754186+0000","last_fullsized":"2024-09-13T14:59:22.754186+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":464,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.749205+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T14:59:22.749205+0000","last_peered":"2024-09-13T14:59:22.749205+0000","last_clean":"2024-09-13T14:59:22.749205+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T14:59:22.749205+0000","last_undegraded":"2024-09-13T14:59:22.749205+0000","last_fullsized":"2024-09-13T14:59:22.749205+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.19","version":"332'8287","reported_seq":8418,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:32.198310+0000","last_change":"2024-09-13T14:57:49.565228+0000","last_active":"2024-09-13T14:59:32.198310+0000","last_peered":"2024-09-13T14:59:32.198310+0000","last_clean":"2024-09-13T14:59:32.198310+0000","last_became_active":"2024-09-13T14:57:42.106480+0000","last_became_peered":"2024-09-13T14:57:42.106480+0000","last_unstale":"2024-09-13T14:59:32.198310+0000","last_undegraded":"2024-09-13T14:59:32.198310+0000","last_fullsized":"2024-09-13T14:59:32.198310+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5099","last_scrub_stamp":"2024-09-13T14:57:49.565078+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.565078+0000","objects_scrubbed":2693,"log_size":8287,"log_dups_size":0,"ondisk_log_size":8287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:01.042206+0000","scrub_duration":773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96600064,"num_objects":1474,"num_object_clones":0,"num_object_copies":4422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1474,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3195,"num_write_kb":63232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":303,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.754603+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:59:22.754603+0000","last_peered":"2024-09-13T14:59:22.754603+0000","last_clean":"2024-09-13T14:59:22.754603+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:59:22.754603+0000","last_undegraded":"2024-09-13T14:59:22.754603+0000","last_fullsized":"2024-09-13T14:59:22.754603+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761860+0000","last_change":"2024-09-13T14:57:23.932331+0000","last_active":"2024-09-13T14:59:21.761860+0000","last_peered":"2024-09-13T14:59:21.761860+0000","last_clean":"2024-09-13T14:59:21.761860+0000","last_became_active":"2024-09-13T14:57:23.932039+0000","last_became_peered":"2024-09-13T14:57:23.932039+0000","last_unstale":"2024-09-13T14:59:21.761860+0000","last_undegraded":"2024-09-13T14:59:21.761860+0000","last_fullsized":"2024-09-13T14:59:21.761860+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:57:40.565036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.18","version":"332'8301","reported_seq":8433,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:31.787270+0000","last_change":"2024-09-13T14:58:00.266091+0000","last_active":"2024-09-13T14:59:31.787270+0000","last_peered":"2024-09-13T14:59:31.787270+0000","last_clean":"2024-09-13T14:57:59.225324+0000","last_became_active":"2024-09-13T14:58:00.266091+0000","last_became_peered":"2024-09-13T14:58:00.266091+0000","last_unstale":"2024-09-13T14:59:31.787270+0000","last_undegraded":"2024-09-13T14:58:00.194448+0000","last_fullsized":"2024-09-13T14:58:00.194336+0000","mapping_epoch":315,"log_start":"301'115","ondisk_log_start":"301'115","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5122","last_scrub_stamp":"2024-09-13T14:57:50.262095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.262095+0000","objects_scrubbed":2756,"log_size":8186,"log_dups_size":0,"ondisk_log_size":8186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:32:36.283129+0000","scrub_duration":540,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100925440,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1540,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3180,"num_write_kb":62848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":1540}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":211,"reported_epoch":331,"state":"active+undersized","last_fresh":"2024-09-13T14:59:21.761333+0000","last_change":"2024-09-13T14:58:00.208196+0000","last_active":"2024-09-13T14:59:21.761333+0000","last_peered":"2024-09-13T14:59:21.761333+0000","last_clean":"2024-09-13T14:40:02.364287+0000","last_became_active":"2024-09-13T14:58:00.208196+0000","last_became_peered":"2024-09-13T14:58:00.208196+0000","last_unstale":"2024-09-13T14:59:21.761333+0000","last_undegraded":"2024-09-13T14:59:21.761333+0000","last_fullsized":"2024-09-13T14:58:00.193143+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:29.409728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":489,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.751676+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T14:59:22.751676+0000","last_peered":"2024-09-13T14:59:22.751676+0000","last_clean":"2024-09-13T14:59:22.751676+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T14:59:22.751676+0000","last_undegraded":"2024-09-13T14:59:22.751676+0000","last_fullsized":"2024-09-13T14:59:22.751676+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"332'8051","reported_seq":8180,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:29.651078+0000","last_change":"2024-09-13T14:57:51.357626+0000","last_active":"2024-09-13T14:59:29.651078+0000","last_peered":"2024-09-13T14:59:29.651078+0000","last_clean":"2024-09-13T14:59:29.651078+0000","last_became_active":"2024-09-13T14:57:42.107486+0000","last_became_peered":"2024-09-13T14:57:42.107486+0000","last_unstale":"2024-09-13T14:59:29.651078+0000","last_undegraded":"2024-09-13T14:59:29.651078+0000","last_fullsized":"2024-09-13T14:59:29.651078+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5002","last_scrub_stamp":"2024-09-13T14:57:51.357494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:51.357494+0000","objects_scrubbed":2703,"log_size":8051,"log_dups_size":0,"ondisk_log_size":8051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:02.249380+0000","scrub_duration":322,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3049,"num_write_kb":60992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":305,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.757158+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:59:21.757158+0000","last_peered":"2024-09-13T14:59:21.757158+0000","last_clean":"2024-09-13T14:59:21.757158+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:59:21.757158+0000","last_undegraded":"2024-09-13T14:59:21.757158+0000","last_fullsized":"2024-09-13T14:59:21.757158+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":536,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.752037+0000","last_change":"2024-09-13T14:58:00.216756+0000","last_active":"2024-09-13T14:59:22.752037+0000","last_peered":"2024-09-13T14:59:22.752037+0000","last_clean":"2024-09-13T14:57:53.144315+0000","last_became_active":"2024-09-13T14:58:00.216756+0000","last_became_peered":"2024-09-13T14:58:00.216756+0000","last_unstale":"2024-09-13T14:59:22.752037+0000","last_undegraded":"2024-09-13T14:59:22.752037+0000","last_fullsized":"2024-09-13T14:58:00.198679+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:29:23.269572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"332'8140","reported_seq":8273,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:29.655456+0000","last_change":"2024-09-13T14:58:00.198384+0000","last_active":"2024-09-13T14:59:29.655456+0000","last_peered":"2024-09-13T14:59:29.655456+0000","last_clean":"2024-09-13T14:57:59.150610+0000","last_became_active":"2024-09-13T14:58:00.198384+0000","last_became_peered":"2024-09-13T14:58:00.198384+0000","last_unstale":"2024-09-13T14:59:29.655456+0000","last_undegraded":"2024-09-13T14:58:00.192532+0000","last_fullsized":"2024-09-13T14:58:00.192417+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"313'5055","last_scrub_stamp":"2024-09-13T14:57:52.447068+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:52.447068+0000","objects_scrubbed":2757,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:25.929132+0000","scrub_duration":434,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100925440,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1540,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3087,"num_write_kb":59840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1540}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":300,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.752500+0000","last_change":"2024-09-13T14:58:00.207661+0000","last_active":"2024-09-13T14:59:22.752500+0000","last_peered":"2024-09-13T14:59:22.752500+0000","last_clean":"2024-09-13T14:40:02.364918+0000","last_became_active":"2024-09-13T14:58:00.207661+0000","last_became_peered":"2024-09-13T14:58:00.207661+0000","last_unstale":"2024-09-13T14:59:22.752500+0000","last_undegraded":"2024-09-13T14:59:22.752500+0000","last_fullsized":"2024-09-13T14:58:00.194016+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:08:37.984416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":501,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.753829+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T14:59:22.753829+0000","last_peered":"2024-09-13T14:59:22.753829+0000","last_clean":"2024-09-13T14:59:22.753829+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T14:59:22.753829+0000","last_undegraded":"2024-09-13T14:59:22.753829+0000","last_fullsized":"2024-09-13T14:59:22.753829+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.15","version":"332'8204","reported_seq":8331,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.757485+0000","last_change":"2024-09-13T14:58:08.102921+0000","last_active":"2024-09-13T14:59:30.757485+0000","last_peered":"2024-09-13T14:59:30.757485+0000","last_clean":"2024-09-13T14:59:30.757485+0000","last_became_active":"2024-09-13T14:57:42.092386+0000","last_became_peered":"2024-09-13T14:57:42.092386+0000","last_unstale":"2024-09-13T14:59:30.757485+0000","last_undegraded":"2024-09-13T14:59:30.757485+0000","last_fullsized":"2024-09-13T14:59:30.757485+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"317'5489","last_scrub_stamp":"2024-09-13T14:58:08.102772+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.102772+0000","objects_scrubbed":3113,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:05.041889+0000","scrub_duration":0.68467245300000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102957056,"num_objects":1571,"num_object_clones":0,"num_object_copies":4713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":37824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":502,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752023+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T14:59:22.752023+0000","last_peered":"2024-09-13T14:59:22.752023+0000","last_clean":"2024-09-13T14:59:22.752023+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T14:59:22.752023+0000","last_undegraded":"2024-09-13T14:59:22.752023+0000","last_fullsized":"2024-09-13T14:59:22.752023+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761611+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T14:59:21.761611+0000","last_peered":"2024-09-13T14:59:21.761611+0000","last_clean":"2024-09-13T14:59:21.761611+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T14:59:21.761611+0000","last_undegraded":"2024-09-13T14:59:21.761611+0000","last_fullsized":"2024-09-13T14:59:21.761611+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:39.699281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"332'8114","reported_seq":8241,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:30.843469+0000","last_change":"2024-09-13T14:58:08.964003+0000","last_active":"2024-09-13T14:59:30.843469+0000","last_peered":"2024-09-13T14:59:30.843469+0000","last_clean":"2024-09-13T14:59:30.843469+0000","last_became_active":"2024-09-13T14:57:42.094348+0000","last_became_peered":"2024-09-13T14:57:42.094348+0000","last_unstale":"2024-09-13T14:59:30.843469+0000","last_undegraded":"2024-09-13T14:59:30.843469+0000","last_fullsized":"2024-09-13T14:59:30.843469+0000","mapping_epoch":307,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"317'5506","last_scrub_stamp":"2024-09-13T14:58:08.963824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.963824+0000","objects_scrubbed":3147,"log_size":8114,"log_dups_size":0,"ondisk_log_size":8114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:21.311000+0000","scrub_duration":0.58457265300000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2620,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":514,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.749694+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T14:59:22.749694+0000","last_peered":"2024-09-13T14:59:22.749694+0000","last_clean":"2024-09-13T14:59:22.749694+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T14:59:22.749694+0000","last_undegraded":"2024-09-13T14:59:22.749694+0000","last_fullsized":"2024-09-13T14:59:22.749694+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":18,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.761094+0000","last_change":"2024-09-13T14:59:13.688428+0000","last_active":"2024-09-13T14:59:21.761094+0000","last_peered":"2024-09-13T14:59:21.761094+0000","last_clean":"2024-09-13T14:59:21.761094+0000","last_became_active":"2024-09-13T14:59:13.687721+0000","last_became_peered":"2024-09-13T14:59:13.687721+0000","last_unstale":"2024-09-13T14:59:21.761094+0000","last_undegraded":"2024-09-13T14:59:21.761094+0000","last_fullsized":"2024-09-13T14:59:21.761094+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:31.018560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.13","version":"332'8310","reported_seq":8442,"reported_epoch":332,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:31.932846+0000","last_change":"2024-09-13T14:58:00.210800+0000","last_active":"2024-09-13T14:59:31.932846+0000","last_peered":"2024-09-13T14:59:31.932846+0000","last_clean":"2024-09-13T14:57:59.137884+0000","last_became_active":"2024-09-13T14:58:00.210800+0000","last_became_peered":"2024-09-13T14:58:00.210800+0000","last_unstale":"2024-09-13T14:59:31.932846+0000","last_undegraded":"2024-09-13T14:58:00.197608+0000","last_fullsized":"2024-09-13T14:58:00.197497+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5142","last_scrub_stamp":"2024-09-13T14:57:50.224834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.224834+0000","objects_scrubbed":2694,"log_size":8310,"log_dups_size":0,"ondisk_log_size":8310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:42.826507+0000","scrub_duration":539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106430464,"num_objects":1624,"num_object_clones":0,"num_object_copies":4872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1624,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3168,"num_write_kb":67136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1624}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760997+0000","last_change":"2024-09-13T14:59:12.712394+0000","last_active":"2024-09-13T14:59:21.760997+0000","last_peered":"2024-09-13T14:59:21.760997+0000","last_clean":"2024-09-13T14:59:21.760997+0000","last_became_active":"2024-09-13T14:59:12.711381+0000","last_became_peered":"2024-09-13T14:59:12.711381+0000","last_unstale":"2024-09-13T14:59:21.760997+0000","last_undegraded":"2024-09-13T14:59:21.760997+0000","last_fullsized":"2024-09-13T14:59:21.760997+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:39:28.567717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":330,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760544+0000","last_change":"2024-09-13T14:59:12.712703+0000","last_active":"2024-09-13T14:59:21.760544+0000","last_peered":"2024-09-13T14:59:21.760544+0000","last_clean":"2024-09-13T14:59:21.760544+0000","last_became_active":"2024-09-13T14:59:12.688810+0000","last_became_peered":"2024-09-13T14:59:12.688810+0000","last_unstale":"2024-09-13T14:59:21.760544+0000","last_undegraded":"2024-09-13T14:59:21.760544+0000","last_fullsized":"2024-09-13T14:59:21.760544+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:51.655399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.12","version":"332'7907","reported_seq":16194,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:31.943370+0000","last_change":"2024-09-13T14:59:22.767421+0000","last_active":"2024-09-13T14:59:31.943370+0000","last_peered":"2024-09-13T14:59:31.943370+0000","last_clean":"2024-09-13T14:59:31.943370+0000","last_became_active":"2024-09-13T14:59:22.756767+0000","last_became_peered":"2024-09-13T14:59:22.756767+0000","last_unstale":"2024-09-13T14:59:31.943370+0000","last_undegraded":"2024-09-13T14:59:31.943370+0000","last_fullsized":"2024-09-13T14:59:31.943370+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"312'4931","last_scrub_stamp":"2024-09-13T14:57:52.187507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:52.187507+0000","objects_scrubbed":2568,"log_size":7907,"log_dups_size":0,"ondisk_log_size":7907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:50.874388+0000","scrub_duration":433,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96993280,"num_objects":1480,"num_object_clones":0,"num_object_copies":4440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2976,"num_write_kb":60416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2715,"num_bytes_recovered":177930240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":331,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:21.761460+0000","last_change":"2024-09-13T14:59:13.726924+0000","last_active":"2024-09-13T14:59:21.761460+0000","last_peered":"2024-09-13T14:59:21.761460+0000","last_clean":"2024-09-13T14:59:21.761460+0000","last_became_active":"2024-09-13T14:59:13.726583+0000","last_became_peered":"2024-09-13T14:59:13.726583+0000","last_unstale":"2024-09-13T14:59:21.761460+0000","last_undegraded":"2024-09-13T14:59:21.761460+0000","last_fullsized":"2024-09-13T14:59:21.761460+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:36.427735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,2],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":315,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.760780+0000","last_change":"2024-09-13T14:59:12.722277+0000","last_active":"2024-09-13T14:59:21.760780+0000","last_peered":"2024-09-13T14:59:21.760780+0000","last_clean":"2024-09-13T14:59:21.760780+0000","last_became_active":"2024-09-13T14:59:12.712060+0000","last_became_peered":"2024-09-13T14:59:12.712060+0000","last_unstale":"2024-09-13T14:59:21.760780+0000","last_undegraded":"2024-09-13T14:59:21.760780+0000","last_fullsized":"2024-09-13T14:59:21.760780+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:42.992912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.11","version":"332'8436","reported_seq":16864,"reported_epoch":332,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T14:59:30.888662+0000","last_change":"2024-09-13T14:59:13.894486+0000","last_active":"2024-09-13T14:59:30.888662+0000","last_peered":"2024-09-13T14:59:30.888662+0000","last_clean":"2024-09-13T14:59:11.679867+0000","last_became_active":"2024-09-13T14:59:13.871230+0000","last_became_peered":"2024-09-13T14:59:13.871230+0000","last_unstale":"2024-09-13T14:59:30.888662+0000","last_undegraded":"2024-09-13T14:59:13.704564+0000","last_fullsized":"2024-09-13T14:59:13.681174+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":8436,"log_dups_size":0,"ondisk_log_size":8436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103579648,"num_objects":1581,"num_object_clones":0,"num_object_copies":4743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3803,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5989,"num_write_kb":242208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2782,"num_bytes_recovered":182190080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,3],"avail_no_missing":["5","2","3","6","7"],"object_location_counts":[{"shards":"2,3,5","objects":1600},{"shards":"3,5","objects":-542}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":538,"reported_epoch":332,"state":"active+undersized","last_fresh":"2024-09-13T14:59:22.752296+0000","last_change":"2024-09-13T14:58:00.209414+0000","last_active":"2024-09-13T14:59:22.752296+0000","last_peered":"2024-09-13T14:59:22.752296+0000","last_clean":"2024-09-13T14:57:53.144644+0000","last_became_active":"2024-09-13T14:58:00.209414+0000","last_became_peered":"2024-09-13T14:58:00.209414+0000","last_unstale":"2024-09-13T14:59:22.752296+0000","last_undegraded":"2024-09-13T14:59:22.752296+0000","last_fullsized":"2024-09-13T14:58:00.192721+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:03:41.274290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":511,"reported_epoch":332,"state":"active+clean","last_fresh":"2024-09-13T14:59:22.752939+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T14:59:22.752939+0000","last_peered":"2024-09-13T14:59:22.752939+0000","last_clean":"2024-09-13T14:59:22.752939+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T14:59:22.752939+0000","last_undegraded":"2024-09-13T14:59:22.752939+0000","last_fullsized":"2024-09-13T14:59:22.752939+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"332'8070","reported_seq":13846,"reported_epoch":332,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:29.689692+0000","last_change":"2024-09-13T14:59:19.419791+0000","last_active":"2024-09-13T14:59:29.689692+0000","last_peered":"2024-09-13T14:59:29.689692+0000","last_clean":"2024-09-13T14:59:29.689692+0000","last_became_active":"2024-09-13T14:58:41.166787+0000","last_became_peered":"2024-09-13T14:58:41.166787+0000","last_unstale":"2024-09-13T14:59:29.689692+0000","last_undegraded":"2024-09-13T14:59:29.689692+0000","last_fullsized":"2024-09-13T14:59:29.689692+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":5,"last_scrub":"330'7721","last_scrub_stamp":"2024-09-13T14:59:19.419459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:59:19.419459+0000","objects_scrubbed":1924,"log_size":8070,"log_dups_size":0,"ondisk_log_size":8070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:51.203762+0000","scrub_duration":933,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1560,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":364,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1560}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":17,"reported_epoch":331,"state":"active+clean","last_fresh":"2024-09-13T14:59:21.761182+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T14:59:21.761182+0000","last_peered":"2024-09-13T14:59:21.761182+0000","last_clean":"2024-09-13T14:59:21.761182+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T14:59:21.761182+0000","last_undegraded":"2024-09-13T14:59:21.761182+0000","last_fullsized":"2024-09-13T14:59:21.761182+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":183,"reported_epoch":331,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:21.761245+0000","last_change":"2024-09-13T14:58:00.200730+0000","last_active":"2024-09-13T14:59:21.761245+0000","last_peered":"2024-09-13T14:59:21.761245+0000","last_clean":"2024-09-13T14:54:54.202829+0000","last_became_active":"2024-09-13T14:58:00.200730+0000","last_became_peered":"2024-09-13T14:58:00.200730+0000","last_unstale":"2024-09-13T14:59:21.761245+0000","last_undegraded":"2024-09-13T14:58:00.190765+0000","last_fullsized":"2024-09-13T14:58:00.190041+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:43.150558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":3189538840,"num_objects":48669,"num_object_clones":0,"num_object_copies":146007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25655,"num_objects_misplaced":5594,"num_objects_unfound":0,"num_objects_dirty":48669,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135835,"num_write_kb":4436449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6369,"num_bytes_recovered":417267712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11424706560,"data_stored":11424694344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203685,"ondisk_log_size":203685,"up":68,"acting":66,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":89,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6346,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":84,"acting":90,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":20,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":29,"acting":30,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234473,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2863476,"kb_used_data":1723180,"kb_used_omap":1,"kb_used_meta":1140222,"kb_avail":90881676,"statfs":{"total":95995035648,"available":93062836224,"internally_reserved":0,"allocated":1764536320,"data_stored":1762714249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1167587787},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496111,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2753324,"kb_used_data":1693476,"kb_used_omap":1,"kb_used_meta":1059838,"kb_avail":90991828,"statfs":{"total":95995035648,"available":93175631872,"internally_reserved":0,"allocated":1734119424,"data_stored":1732305484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1085274567},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757749,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1751588,"kb_used_data":601500,"kb_used_omap":1,"kb_used_meta":1150078,"kb_avail":91993564,"statfs":{"total":95995035648,"available":94201409536,"internally_reserved":0,"allocated":615936000,"data_stored":614126291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1177680326},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019385,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1256240,"kb_used_data":318632,"kb_used_omap":1,"kb_used_meta":937598,"kb_avail":92488912,"statfs":{"total":95995035648,"available":94708645888,"internally_reserved":0,"allocated":326279168,"data_stored":324461189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":960100806},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650597,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2503712,"kb_used_data":1768408,"kb_used_omap":4,"kb_used_meta":735291,"kb_avail":91241440,"statfs":{"total":95995035648,"available":93431234560,"internally_reserved":0,"allocated":1810849792,"data_stored":1809043941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":752938543},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879532,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2746028,"kb_used_data":2048740,"kb_used_omap":6,"kb_used_meta":697209,"kb_avail":90999124,"statfs":{"total":95995035648,"available":93183102976,"internally_reserved":0,"allocated":2097909760,"data_stored":2096092261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":713942585},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,2,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435548,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2204444,"kb_used_data":1206036,"kb_used_omap":5,"kb_used_meta":998394,"kb_avail":91540708,"statfs":{"total":95995035648,"available":93737684992,"internally_reserved":0,"allocated":1234980864,"data_stored":1233177994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1022355809},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926105,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2675416,"kb_used_data":2196816,"kb_used_omap":5,"kb_used_meta":478586,"kb_avail":91069736,"statfs":{"total":95995035648,"available":93255409664,"internally_reserved":0,"allocated":2249539584,"data_stored":2247846175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":490072234},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2269839360,"data_stored":2269839360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1224937472,"data_stored":1224933400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2079526912,"data_stored":2079522840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1773797376,"data_stored":1773797376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":19857408,"data_stored":19857408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":602013696,"data_stored":602013696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1717633024,"data_stored":1717633024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1737101312,"data_stored":1737097240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T14:59:38.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-13T14:59:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.107 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:38.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:38.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:38.233 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:37 smithi159 ceph-mon[102519]: pgmap v1188: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 25013/137574 objects degraded (18.181%); 5430/137574 objects misplaced (3.947%); 2.7 MiB/s, 43 objects/s recovering 2024-09-13T14:59:38.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:38.398 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:38.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:38.557 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:38.660 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:38.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:38.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:38.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T14:59:38.730 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:38.788 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:38.839 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:38.866 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:38.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:38.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:39.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:39.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:39.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:39.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:39.289 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:39.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:39.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:39.436 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:39.538 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:39.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:39.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:39.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:39.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:39.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:39.823 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T14:59:39.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:39.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:39.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:39.975 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:39.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:40.077 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:40.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:39] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-13T14:59:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:39 smithi042 ceph-mon[113926]: pgmap v1189: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23713/130110 objects degraded (18.225%); 4861/130110 objects misplaced (3.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:59:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:39 smithi042 ceph-mon[108570]: pgmap v1189: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23713/130110 objects degraded (18.225%); 4861/130110 objects misplaced (3.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:59:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:40.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.105 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:40.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:40.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:40.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:40.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:39 smithi159 ceph-mon[102519]: pgmap v1189: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23713/130110 objects degraded (18.225%); 4861/130110 objects misplaced (3.736%); 4.1 MiB/s, 65 objects/s recovering 2024-09-13T14:59:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T14:59:40.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:40.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:40.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:40.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.532 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:40.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:40.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:40.674 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:40.776 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:40.821 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:40.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:40.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:40.946 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:40.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:41.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[108570]: osdmap e333: 8 total, 7 up, 8 in 2024-09-13T14:59:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 23713/130110 objects degraded (18.225%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:41.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:41.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:41.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[113926]: osdmap e333: 8 total, 7 up, 8 in 2024-09-13T14:59:41.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 23713/130110 objects degraded (18.225%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:41.179 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:40 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T14:59:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2691670169' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T14:59:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:40 smithi159 ceph-mon[102519]: osdmap e333: 8 total, 7 up, 8 in 2024-09-13T14:59:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 23713/130110 objects degraded (18.225%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:41.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:41.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:41.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:41.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:41.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:41.606 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:41.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:41.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:41.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:41.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:41.964 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:41.990 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:41.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:41.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:42.092 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:41 smithi042 ceph-mon[108570]: pgmap v1191: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23403/126837 objects degraded (18.451%); 4729/126837 objects misplaced (3.728%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T14:59:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:41 smithi042 ceph-mon[113926]: pgmap v1191: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23403/126837 objects degraded (18.451%); 4729/126837 objects misplaced (3.728%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T14:59:42.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:42.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:42.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:42.248 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:41 smithi159 ceph-mon[102519]: pgmap v1191: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 23403/126837 objects degraded (18.451%); 4729/126837 objects misplaced (3.728%); 3.2 MiB/s, 50 objects/s recovering 2024-09-13T14:59:42.265 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:42.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:42.406 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:42.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:42.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:42.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:42.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:42.659 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:42.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:42.790 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:42.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:42.892 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:42.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:42.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:43.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:43.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:43.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:43.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:43.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:43.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:43.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:43.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:43.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:43.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:43.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:43.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.725 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:43.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:43.867 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:43.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:43.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:43.990 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:43.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:44.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:43 smithi042 ceph-mon[108570]: pgmap v1192: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:44.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:43 smithi042 ceph-mon[113926]: pgmap v1192: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:44.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:43 smithi159 ceph-mon[102519]: pgmap v1192: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:44.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:44.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:44.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:44.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:44.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:44.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:44.626 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:44.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:44.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:44.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:44.780 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:44.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:44.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:44.924 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:44.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:45.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:45.156 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:45.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:45.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:45.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:45.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:45.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:45.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:45.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:45.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:45.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:45.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:45.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:45.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:46.072 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:46.087 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:45 smithi159 ceph-mon[102519]: pgmap v1193: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:46.087 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 21735/114621 objects degraded (18.962%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:45 smithi042 ceph-mon[108570]: pgmap v1193: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 21735/114621 objects degraded (18.962%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:46.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:45 smithi042 ceph-mon[113926]: pgmap v1193: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 21735/114621 objects degraded (18.962%); 4120/114621 objects misplaced (3.594%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:46.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 21735/114621 objects degraded (18.962%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.104 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:46.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:46.205 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:46.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:46.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:46.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:46.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:46.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.503 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:46.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:46.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:46.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:46.743 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:46.780 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:46.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:46.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:46.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:46.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:47.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:47.059 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:47.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:47.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:47.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:47.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:47.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:47.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:47.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:47.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:47.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:47.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:47.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:47.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:47.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:48.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:48.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:48.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:48.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:47 smithi042 ceph-mon[108570]: pgmap v1194: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 21084/106710 objects degraded (19.758%); 3971/106710 objects misplaced (3.721%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:48.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:47 smithi042 ceph-mon[113926]: pgmap v1194: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 21084/106710 objects degraded (19.758%); 3971/106710 objects misplaced (3.721%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:48.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:48.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:47 smithi159 ceph-mon[102519]: pgmap v1194: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 21084/106710 objects degraded (19.758%); 3971/106710 objects misplaced (3.721%); 3.5 MiB/s, 55 objects/s recovering 2024-09-13T14:59:48.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:48.315 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:48.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.445 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:48.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:48.574 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:48.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:48.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:48.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:48.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:48.935 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:48.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:48.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:49.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.227 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:49.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:49.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:49.370 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:49.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:49.471 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:49.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:49.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T14:59:49.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:49.638 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:49.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:49.740 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:49.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.767 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:49.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:49.840 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:49] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-13T14:59:49.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:49.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:49.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:49.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:49.998 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:50.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:49 smithi042 ceph-mon[113926]: pgmap v1195: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 19834/99939 objects degraded (19.846%); 3330/99939 objects misplaced (3.332%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T14:59:50.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:49 smithi042 ceph-mon[108570]: pgmap v1195: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 19834/99939 objects degraded (19.846%); 3330/99939 objects misplaced (3.332%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T14:59:50.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:50.173 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:50.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:49 smithi159 ceph-mon[102519]: pgmap v1195: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s; 19834/99939 objects degraded (19.846%); 3330/99939 objects misplaced (3.332%); 3.9 MiB/s, 63 objects/s recovering 2024-09-13T14:59:50.275 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:50.406 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:50.432 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:50.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:50.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:50.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:50.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:50.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:50.919 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 1 pgp_num to 11 2024-09-13T14:59:50.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:50.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:50.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:51.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:59:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 19834/99939 objects degraded (19.846%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 19834/99939 objects degraded (19.846%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:51.203 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.237 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:51.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-13T14:59:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 19834/99939 objects degraded (19.846%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T14:59:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T14:59:51.269 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T14:59:51.270 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T14:59:51.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:51.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:51.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:51.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:51.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:51.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:51.640 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:51.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:51.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:51.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:51.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:51.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:51.912 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:51.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:52.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.046 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.148 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:52.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:51 smithi159 ceph-mon[102519]: pgmap v1196: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19522/96612 objects degraded (20.207%); 3171/96612 objects misplaced (3.282%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:59:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:59:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:51 smithi159 ceph-mon[102519]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:59:52.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:52.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:52.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[113926]: pgmap v1196: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19522/96612 objects degraded (20.207%); 3171/96612 objects misplaced (3.282%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:59:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:59:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[113926]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:59:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[108570]: pgmap v1196: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 19522/96612 objects degraded (20.207%); 3171/96612 objects misplaced (3.282%); 3.7 MiB/s, 58 objects/s recovering 2024-09-13T14:59:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3622996773' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-13T14:59:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:51 smithi042 ceph-mon[108570]: osdmap e334: 8 total, 7 up, 8 in 2024-09-13T14:59:52.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:52.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:52.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:52.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.691 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:52.730 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:52.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:52.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:52.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:52.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:53.010 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:53.029 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:53.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.289 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:53.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:53.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:53.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:53.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:53.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:53.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:53.719 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:53.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.860 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:53.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:53.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:53.996 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:53.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:53.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:54.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:54.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:54.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:53 smithi159 ceph-mon[102519]: pgmap v1198: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:54.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:54.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:54.325 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:53 smithi042 ceph-mon[113926]: pgmap v1198: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:54.327 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:53 smithi042 ceph-mon[108570]: pgmap v1198: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:54.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T14:59:54.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:54.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:54.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:54.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:54.653 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:54.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:54.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:54.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:54.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:54.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:54.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:54.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:55.016 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:55.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:55.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:55.213 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:55.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:55.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:55.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:55.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:55.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:55.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:55.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:55.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:55.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:55.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:55.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:55.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:56.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:55 smithi042 ceph-mon[108570]: pgmap v1199: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:56.061 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 17801/84141 objects degraded (21.156%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:56.061 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:55 smithi042 ceph-mon[113926]: pgmap v1199: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:56.061 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 17801/84141 objects degraded (21.156%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:56.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:56.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:56.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:55 smithi159 ceph-mon[102519]: pgmap v1199: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 17801/84141 objects degraded (21.156%); 2581/84141 objects misplaced (3.067%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 17801/84141 objects degraded (21.156%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T14:59:56.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:56.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:56.476 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:56.578 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:56.618 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:56.720 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:56.747 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:56.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:56.849 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:56.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:56.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:56.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:56.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:57.013 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:57.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:57.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:57.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:57.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:57.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:57.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:57.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:57.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:57.453 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T14:59:57.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:57.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:57.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:57.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:57.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:57.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:57.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T14:59:57.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:57.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T14:59:58.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.079 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:58.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.181 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.188 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:57 smithi042 ceph-mon[113926]: pgmap v1200: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 17192/75978 objects degraded (22.628%); 2430/75978 objects misplaced (3.198%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:58.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:57 smithi042 ceph-mon[108570]: pgmap v1200: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 17192/75978 objects degraded (22.628%); 2430/75978 objects misplaced (3.198%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:58.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:57 smithi159 ceph-mon[102519]: pgmap v1200: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 17192/75978 objects degraded (22.628%); 2430/75978 objects misplaced (3.198%); 3.5 MiB/s, 68 objects/s recovering 2024-09-13T14:59:58.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:58.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:58.463 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:58.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:58.565 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T14:59:58.594 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T14:59:58.607 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:58.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T14:59:58.639 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T14:59:58.640 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T14:59:58.684 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T14:59:58.684 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T14:59:58.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T14:59:58.722 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T14:59:58.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T14:59:58.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:58.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T14:59:58.754 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T14:59:58.754 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T14:59:58.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T14:59:58.788 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T14:59:58.821 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T14:59:58.822 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T14:59:58.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:58.857 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T14:59:58.858 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T14:59:58.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:58.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:58.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:58.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T14:59:59.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T14:59:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2560753899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2560753899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:59.105 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.161 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:59.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2560753899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T14:59:59.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:59.389 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T14:59:59.433 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T14:59:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T14:59:59.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:59.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T14:59:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T14:59:59.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:59.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:59.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T14:59:59.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T14:59:59.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T14:59:59.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T14:59:59.956 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T14:59:59.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[113926]: pgmap v1201: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 15853/68562 objects degraded (23.122%); 2107/68562 objects misplaced (3.073%); 1.4 MiB/s, 79 objects/s recovering 2024-09-13T14:59:59.982 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1909139115' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:59.983 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:59.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[108570]: pgmap v1201: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 15853/68562 objects degraded (23.122%); 2107/68562 objects misplaced (3.073%); 1.4 MiB/s, 79 objects/s recovering 2024-09-13T14:59:59.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1909139115' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:59.983 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T14:59:59.983 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 14:59:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:14:59:59] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.51.0" 2024-09-13T15:00:00.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:00.044 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T15:00:00.044 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:00.045 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_7 2024-09-13T15:00:00.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:00:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:00.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:00.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:59 smithi159 ceph-mon[102519]: pgmap v1201: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 15853/68562 objects degraded (23.122%); 2107/68562 objects misplaced (3.073%); 1.4 MiB/s, 79 objects/s recovering 2024-09-13T15:00:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1909139115' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 14:59:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:00:00.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:00.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:00.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:00.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:00.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:00.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:00.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:00.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:00.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:00.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:00.952 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:00.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:00.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:01.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:01.121 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:01.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: osd.0 (root=default,host=smithi042) is down 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: [WRN] PG_DEGRADED: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 1.0 is stuck undersized for 118s, current state active+undersized, last acting [6,7] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 1.1 is stuck undersized for 118s, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 1.6 is stuck undersized for 118s, current state active+undersized, last acting [6,3] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 2.0 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 2.1 is stuck undersized for 118s, current state active+undersized, last acting [2,3] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 2.4 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.7 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.10 is stuck undersized for 118s, current state active+undersized+degraded, last acting [6,5] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.12 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.17 is stuck undersized for 118s, current state active+undersized, last acting [5,3] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.18 is stuck undersized for 118s, current state active+undersized, last acting [3,1] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 3.1f is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.0 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.3 is stuck undersized for 118s, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.6 is stuck undersized for 118s, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.7 is stuck undersized for 118s, current state active+undersized, last acting [1,5] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.16 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.19 is stuck undersized for 118s, current state active+undersized, last acting [3,2] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 4.1e is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.0 is active+recovering+undersized+degraded+remapped, acting [2,6] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.3 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.6 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.8 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.a is stuck undersized for 118s, current state active+undersized+degraded, last acting [2,3] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.c is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.e is active+undersized+degraded+remapped+backfill_wait, acting [3,7] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.11 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.13 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.16 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: pg 12.18 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.252 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:01.260 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: osd.0 (root=default,host=smithi042) is down 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: [WRN] PG_DEGRADED: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 1.0 is stuck undersized for 118s, current state active+undersized, last acting [6,7] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 1.1 is stuck undersized for 118s, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 1.6 is stuck undersized for 118s, current state active+undersized, last acting [6,3] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 2.0 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 2.1 is stuck undersized for 118s, current state active+undersized, last acting [2,3] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 2.4 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.7 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.10 is stuck undersized for 118s, current state active+undersized+degraded, last acting [6,5] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.12 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.17 is stuck undersized for 118s, current state active+undersized, last acting [5,3] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.18 is stuck undersized for 118s, current state active+undersized, last acting [3,1] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 3.1f is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.0 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.3 is stuck undersized for 118s, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.6 is stuck undersized for 118s, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.7 is stuck undersized for 118s, current state active+undersized, last acting [1,5] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.16 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.19 is stuck undersized for 118s, current state active+undersized, last acting [3,2] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 4.1e is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.0 is active+recovering+undersized+degraded+remapped, acting [2,6] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-13T15:00:01.351 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.3 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.6 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.8 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.a is stuck undersized for 118s, current state active+undersized+degraded, last acting [2,3] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.c is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.e is active+undersized+degraded+remapped+backfill_wait, acting [3,7] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.11 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.13 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.16 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: pg 12.18 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.352 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: osdmap e335: 8 total, 7 up, 8 in 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: [WRN] OSD_DOWN: 1 osds down 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: osd.0 (root=default,host=smithi042) is down 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: [WRN] PG_DEGRADED: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized 2024-09-13T15:00:01.353 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 1.0 is stuck undersized for 118s, current state active+undersized, last acting [6,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 1.1 is stuck undersized for 118s, current state active+undersized, last acting [5,2] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 1.6 is stuck undersized for 118s, current state active+undersized, last acting [6,3] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 2.0 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 2.1 is stuck undersized for 118s, current state active+undersized, last acting [2,3] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 2.4 is stuck undersized for 118s, current state active+undersized, last acting [1,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.7 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.10 is stuck undersized for 118s, current state active+undersized+degraded, last acting [6,5] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.12 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.17 is stuck undersized for 118s, current state active+undersized, last acting [5,3] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.18 is stuck undersized for 118s, current state active+undersized, last acting [3,1] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 3.1f is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.0 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.354 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.3 is stuck undersized for 118s, current state active+undersized, last acting [5,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.6 is stuck undersized for 118s, current state active+undersized, last acting [1,2] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.7 is stuck undersized for 118s, current state active+undersized, last acting [1,5] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.16 is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.19 is stuck undersized for 118s, current state active+undersized, last acting [3,2] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 4.1e is stuck undersized for 118s, current state active+undersized, last acting [3,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.0 is active+recovering+undersized+degraded+remapped, acting [2,6] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.3 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.6 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.8 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.a is stuck undersized for 118s, current state active+undersized+degraded, last acting [2,3] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.c is stuck undersized for 118s, current state active+undersized+degraded, last acting [5,2] 2024-09-13T15:00:01.355 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.e is active+undersized+degraded+remapped+backfill_wait, acting [3,7] 2024-09-13T15:00:01.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.11 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2024-09-13T15:00:01.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.13 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,7] 2024-09-13T15:00:01.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.16 is stuck undersized for 118s, current state active+undersized+degraded, last acting [1,7] 2024-09-13T15:00:01.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: pg 12.18 is stuck undersized for 118s, current state active+undersized+degraded, last acting [3,2] 2024-09-13T15:00:01.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 15853/68562 objects degraded (23.122%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:01.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:01.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:01.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:01.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:01.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:01.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:01.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:01.759 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:01.781 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:01.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:01.883 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:01.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:01.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:02.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:02.049 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:02.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:02.151 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:02.181 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:02.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:01 smithi159 ceph-mon[102519]: pgmap v1203: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 15560/65487 objects degraded (23.760%); 1961/65487 objects misplaced (2.994%); 1.5 MiB/s, 82 objects/s recovering 2024-09-13T15:00:02.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:02.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:01 smithi042 ceph-mon[108570]: pgmap v1203: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 15560/65487 objects degraded (23.760%); 1961/65487 objects misplaced (2.994%); 1.5 MiB/s, 82 objects/s recovering 2024-09-13T15:00:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:01 smithi042 ceph-mon[113926]: pgmap v1203: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 15560/65487 objects degraded (23.760%); 1961/65487 objects misplaced (2.994%); 1.5 MiB/s, 82 objects/s recovering 2024-09-13T15:00:02.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:02.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:02.549 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:02.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:02.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:02.721 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:02.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:02.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:02.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:02.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:02.989 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:02.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:02.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:03.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:03.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:03.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:03.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:03.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:03.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:03.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.380 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:03.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:03.481 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:03.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:03.663 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:03.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:03.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:03.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:03.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:03.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:03.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:04.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:04.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:04.185 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:04.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:03 smithi159 ceph-mon[102519]: pgmap v1204: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:04.316 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:03 smithi042 ceph-mon[113926]: pgmap v1204: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:03 smithi042 ceph-mon[108570]: pgmap v1204: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:04.349 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:04.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:04.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:04.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:04.597 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:04.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.651 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:04.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:04.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:04.796 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:04.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:04.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:04.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:04.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:05.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:05.078 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:05.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:05.410 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:05.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:05.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:05.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:05.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:05.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:05.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.820 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:05.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:05.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:05.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:05.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:06.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.078 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:06.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[113926]: pgmap v1205: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 14132/54966 objects degraded (25.710%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[108570]: pgmap v1205: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 14132/54966 objects degraded (25.710%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.180 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:06.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:06.218 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:05 smithi159 ceph-mon[102519]: pgmap v1205: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 1.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 971 op/s; 14132/54966 objects degraded (25.710%); 1689/54966 objects misplaced (3.073%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 14132/54966 objects degraded (25.710%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:06.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:06.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.493 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:06.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:06.572 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:00:06.572 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:00:06.572 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:00:06.572 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:00:06.572 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:00:06.594 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:06.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:06.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:06.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:06.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:06.783 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:06.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:06.915 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:06.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:07.016 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:07.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:07.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:07.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:07.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:07.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:07.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:07.412 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:07.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:07.444 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:07.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:07.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:07.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:07.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:07.695 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:07.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:07.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:00:07.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:07.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:07.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:07.829 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:07.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:07.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:07.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:07.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:08.044 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:08.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:08.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:07 smithi042 ceph-mon[113926]: pgmap v1206: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 981 MiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 13520/47154 objects degraded (28.672%); 1534/47154 objects misplaced (3.253%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:08.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:07 smithi042 ceph-mon[113926]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T15:00:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:07 smithi042 ceph-mon[108570]: pgmap v1206: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 981 MiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 13520/47154 objects degraded (28.672%); 1534/47154 objects misplaced (3.253%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:08.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:07 smithi042 ceph-mon[108570]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T15:00:08.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:08.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:08.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:07 smithi159 ceph-mon[102519]: pgmap v1206: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 981 MiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 13520/47154 objects degraded (28.672%); 1534/47154 objects misplaced (3.253%); 0 B/s, 97 objects/s recovering 2024-09-13T15:00:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:07 smithi159 ceph-mon[102519]: osdmap e336: 8 total, 7 up, 8 in 2024-09-13T15:00:08.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:08.348 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:08.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:08.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:00:08.449 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:08.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:08.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:08.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:08.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:08.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:08.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:08.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:08.910 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:08.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:09.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:09.055 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:09.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:09.190 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:09.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:09.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:09.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:09.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:09 smithi042 ceph-mon[108570]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T15:00:09.358 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:09 smithi042 ceph-mon[113926]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T15:00:09.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:09.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:09 smithi159 ceph-mon[102519]: osdmap e337: 8 total, 7 up, 8 in 2024-09-13T15:00:09.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:09.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:09.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:09.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:09.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:09.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:09.869 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:09.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:09.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:09.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:10.000 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:10.015 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:09] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-09-13T15:00:10.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:10.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:10.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:10.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:10.274 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:10.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:10 smithi042 ceph-mon[108570]: pgmap v1209: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 837 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 12193/40224 objects degraded (30.313%); 1227/40224 objects misplaced (3.050%); 0 B/s, 65 objects/s recovering 2024-09-13T15:00:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:10 smithi042 ceph-mon[113926]: pgmap v1209: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 837 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 12193/40224 objects degraded (30.313%); 1227/40224 objects misplaced (3.050%); 0 B/s, 65 objects/s recovering 2024-09-13T15:00:10.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:10.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.415 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:10.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:10 smithi159 ceph-mon[102519]: pgmap v1209: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 837 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 12193/40224 objects degraded (30.313%); 1227/40224 objects misplaced (3.050%); 0 B/s, 65 objects/s recovering 2024-09-13T15:00:10.516 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:10.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:10.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:10.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:10.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:10.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:10.822 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:10.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:10.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:10.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:10.960 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:11.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:11.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:11.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:11.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:11.270 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:11 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 12193/40224 objects degraded (30.313%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:11.270 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:11 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 12193/40224 objects degraded (30.313%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:11.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:11.391 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:11.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:11.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:11.498 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:11 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 12193/40224 objects degraded (30.313%), 15 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:11.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:11.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:11.649 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:11.741 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T15:00:11.751 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:11.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:11.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:00:11.796 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:11.845 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:00:11.845 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:11.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:11.891 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:00:11.892 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:00:11.910 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:11.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:11.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:11.939 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:00:11.939 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:11.974 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:00:11.975 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:12.012 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:12.016 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:00:12.016 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:12.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:12.056 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:00:12.057 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:12.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:12 smithi042 ceph-mon[113926]: pgmap v1210: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 773 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 11896/37149 objects degraded (32.022%); 1074/37149 objects misplaced (2.891%) 2024-09-13T15:00:12.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1101948067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:12.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:12 smithi042 ceph-mon[108570]: pgmap v1210: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 773 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 11896/37149 objects degraded (32.022%); 1074/37149 objects misplaced (2.891%) 2024-09-13T15:00:12.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1101948067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:12.141 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:12.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:12.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:12.272 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.311 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:12.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:12.368 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:00:12.370 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:00:12.412 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:12.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:12.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:12 smithi159 ceph-mon[102519]: pgmap v1210: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 67 active+clean; 773 MiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 742 op/s; 11896/37149 objects degraded (32.022%); 1074/37149 objects misplaced (2.891%) 2024-09-13T15:00:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1101948067' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:12.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:12.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:12.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:12.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:12.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:12.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:12.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:12.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:12.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:12.999 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:13.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:13.073 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:13.073 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:13.101 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:13.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:13.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:13.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:13.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:13 smithi042 ceph-mon[108570]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:13.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:13 smithi042 ceph-mon[113926]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:13.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:13.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:13.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:13 smithi159 ceph-mon[102519]: from='client.45713 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:00:13.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:13.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:13.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:13.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:13.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:13.802 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:13.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.841 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:13.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:13.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:13.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:13.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:13.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:14.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:14.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:14.166 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:14.223 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:14.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.269 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:14.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:14.287 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:14 smithi042 ceph-mon[113926]: pgmap v1211: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:14.287 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:14 smithi042 ceph-mon[108570]: pgmap v1211: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:14.371 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:14.412 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:14.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:14 smithi159 ceph-mon[102519]: pgmap v1211: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.22k op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:14.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:14.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:14.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:14.751 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1210,"stamp":"2024-09-13T15:00:10.589521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":810391549,"num_objects":12383,"num_object_clones":0,"num_object_copies":37149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11896,"num_objects_misplaced":1074,"num_objects_unfound":0,"num_objects_dirty":12383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172211,"num_write_kb":4436482,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8640,"num_bytes_recovered":490936522,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203977,"ondisk_log_size":203977,"up":288,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12715940,"kb_used_data":5325796,"kb_used_omap":28,"kb_used_meta":7389987,"kb_avail":737245276,"statfs":{"total":767960285184,"available":754939162624,"internally_reserved":0,"allocated":5453615104,"data_stored":5439060511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":29295,"internal_metadata":7567347089},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,5,6,11,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-389218304,"num_objects":-5939,"num_object_clones":0,"num_object_copies":-17817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2236,"num_objects_misplaced":-615,"num_objects_unfound":0,"num_objects_dirty":-5939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5939,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003942"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":497,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.004566+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:00:07.004566+0000","last_peered":"2024-09-13T15:00:07.004566+0000","last_clean":"2024-09-13T15:00:07.004566+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:00:07.004566+0000","last_undegraded":"2024-09-13T15:00:07.004566+0000","last_fullsized":"2024-09-13T15:00:07.004566+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":26,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941175+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T14:59:50.941175+0000","last_peered":"2024-09-13T14:59:50.941175+0000","last_clean":"2024-09-13T14:59:50.941175+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T14:59:50.941175+0000","last_undegraded":"2024-09-13T14:59:50.941175+0000","last_fullsized":"2024-09-13T14:59:50.941175+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919563+0000","last_change":"2024-09-13T14:40:37.182774+0000","last_active":"2024-09-13T14:59:50.919563+0000","last_peered":"2024-09-13T14:59:50.919563+0000","last_clean":"2024-09-13T14:59:50.919563+0000","last_became_active":"2024-09-13T14:40:37.182379+0000","last_became_peered":"2024-09-13T14:40:37.182379+0000","last_unstale":"2024-09-13T14:59:50.919563+0000","last_undegraded":"2024-09-13T14:59:50.919563+0000","last_fullsized":"2024-09-13T14:59:50.919563+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:05:34.461491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.011220+0000","last_change":"2024-09-13T14:58:00.207288+0000","last_active":"2024-09-13T15:00:07.011220+0000","last_peered":"2024-09-13T15:00:07.011220+0000","last_clean":"2024-09-13T14:40:02.364020+0000","last_became_active":"2024-09-13T14:58:00.207288+0000","last_became_peered":"2024-09-13T14:58:00.207288+0000","last_unstale":"2024-09-13T15:00:07.011220+0000","last_undegraded":"2024-09-13T15:00:07.011220+0000","last_fullsized":"2024-09-13T14:58:00.194753+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:56:13.138124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":308,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.005584+0000","last_change":"2024-09-13T14:58:00.207997+0000","last_active":"2024-09-13T15:00:07.005584+0000","last_peered":"2024-09-13T15:00:07.005584+0000","last_clean":"2024-09-13T14:40:02.365015+0000","last_became_active":"2024-09-13T14:58:00.207997+0000","last_became_peered":"2024-09-13T14:58:00.207997+0000","last_unstale":"2024-09-13T15:00:07.005584+0000","last_undegraded":"2024-09-13T15:00:07.005584+0000","last_fullsized":"2024-09-13T14:58:00.194590+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:03.388390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940596+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T14:59:50.940596+0000","last_peered":"2024-09-13T14:59:50.940596+0000","last_clean":"2024-09-13T14:59:50.940596+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T14:59:50.940596+0000","last_undegraded":"2024-09-13T14:59:50.940596+0000","last_fullsized":"2024-09-13T14:59:50.940596+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:22:05.891150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":523,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.004615+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:00:07.004615+0000","last_peered":"2024-09-13T15:00:07.004615+0000","last_clean":"2024-09-13T15:00:07.004615+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:00:07.004615+0000","last_undegraded":"2024-09-13T15:00:07.004615+0000","last_fullsized":"2024-09-13T15:00:07.004615+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":397,"reported_epoch":334,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:50.927804+0000","last_change":"2024-09-13T14:58:00.194821+0000","last_active":"2024-09-13T14:59:50.927804+0000","last_peered":"2024-09-13T14:59:50.927804+0000","last_clean":"2024-09-13T14:57:53.238105+0000","last_became_active":"2024-09-13T14:58:00.194821+0000","last_became_peered":"2024-09-13T14:58:00.194821+0000","last_unstale":"2024-09-13T14:59:50.927804+0000","last_undegraded":"2024-09-13T14:58:00.191302+0000","last_fullsized":"2024-09-13T14:58:00.191197+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:31:09.722082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.0","version":"335'8882","reported_seq":15130,"reported_epoch":335,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:02.370607+0000","last_change":"2024-09-13T14:59:13.904786+0000","last_active":"2024-09-13T15:00:02.370607+0000","last_peered":"2024-09-13T15:00:02.370607+0000","last_clean":"2024-09-13T14:59:11.474663+0000","last_became_active":"2024-09-13T14:59:13.846462+0000","last_became_peered":"2024-09-13T14:59:13.846462+0000","last_unstale":"2024-09-13T15:00:02.370607+0000","last_undegraded":"2024-09-13T14:59:13.701425+0000","last_fullsized":"2024-09-13T14:59:13.678382+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":8882,"log_dups_size":0,"ondisk_log_size":8882,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40402944,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6564,"num_write_kb":229792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3086,"num_bytes_recovered":130809856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[2,6],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6","objects":610}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":25,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.942213+0000","last_change":"2024-09-13T14:59:13.695251+0000","last_active":"2024-09-13T14:59:50.942213+0000","last_peered":"2024-09-13T14:59:50.942213+0000","last_clean":"2024-09-13T14:59:50.942213+0000","last_became_active":"2024-09-13T14:59:13.687274+0000","last_became_peered":"2024-09-13T14:59:13.687274+0000","last_unstale":"2024-09-13T14:59:50.942213+0000","last_undegraded":"2024-09-13T14:59:50.942213+0000","last_fullsized":"2024-09-13T14:59:50.942213+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:45:08.486245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":292,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919257+0000","last_change":"2024-09-13T14:20:46.006951+0000","last_active":"2024-09-13T14:59:50.919257+0000","last_peered":"2024-09-13T14:59:50.919257+0000","last_clean":"2024-09-13T14:59:50.919257+0000","last_became_active":"2024-09-13T14:20:46.006619+0000","last_became_peered":"2024-09-13T14:20:46.006619+0000","last_unstale":"2024-09-13T14:59:50.919257+0000","last_undegraded":"2024-09-13T14:59:50.919257+0000","last_fullsized":"2024-09-13T14:59:50.919257+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:24:48.358897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.1","version":"335'9358","reported_seq":9418,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.838277+0000","last_change":"2024-09-13T14:58:32.581659+0000","last_active":"2024-09-13T15:00:05.838277+0000","last_peered":"2024-09-13T15:00:05.838277+0000","last_clean":"2024-09-13T15:00:05.838277+0000","last_became_active":"2024-09-13T14:57:42.090080+0000","last_became_peered":"2024-09-13T14:57:42.090080+0000","last_unstale":"2024-09-13T15:00:05.838277+0000","last_undegraded":"2024-09-13T15:00:05.838277+0000","last_fullsized":"2024-09-13T15:00:05.838277+0000","mapping_epoch":298,"log_start":"301'720","ondisk_log_start":"301'720","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":8638,"log_dups_size":0,"ondisk_log_size":8638,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28606464,"num_objects":436,"num_object_clones":0,"num_object_copies":1308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6910,"num_write_kb":235104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":24,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940903+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T14:59:50.940903+0000","last_peered":"2024-09-13T14:59:50.940903+0000","last_clean":"2024-09-13T14:59:50.940903+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T14:59:50.940903+0000","last_undegraded":"2024-09-13T14:59:50.940903+0000","last_fullsized":"2024-09-13T14:59:50.940903+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:40:49.409549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940774+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T14:59:50.940774+0000","last_peered":"2024-09-13T14:59:50.940774+0000","last_clean":"2024-09-13T14:59:50.940774+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T14:59:50.940774+0000","last_undegraded":"2024-09-13T14:59:50.940774+0000","last_fullsized":"2024-09-13T14:59:50.940774+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.2","version":"335'9130","reported_seq":9268,"reported_epoch":336,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:07.012360+0000","last_change":"2024-09-13T14:59:13.904720+0000","last_active":"2024-09-13T15:00:07.012360+0000","last_peered":"2024-09-13T15:00:07.012360+0000","last_clean":"2024-09-13T14:59:11.647182+0000","last_became_active":"2024-09-13T14:59:13.880217+0000","last_became_peered":"2024-09-13T14:59:13.880217+0000","last_unstale":"2024-09-13T15:00:07.012360+0000","last_undegraded":"2024-09-13T14:59:13.709724+0000","last_fullsized":"2024-09-13T14:59:13.682580+0000","mapping_epoch":326,"log_start":"302'1923","ondisk_log_start":"302'1923","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'4893","last_scrub_stamp":"2024-09-13T14:57:49.600786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.600786+0000","objects_scrubbed":2644,"log_size":7207,"log_dups_size":0,"ondisk_log_size":7207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":823,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4245,"num_write_kb":63424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[3,7],"avail_no_missing":["3","6","7"],"object_location_counts":[{"shards":"3,6,7","objects":381},{"shards":"3,7","objects":-1660}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":209,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919420+0000","last_change":"2024-09-13T14:43:35.907433+0000","last_active":"2024-09-13T14:59:50.919420+0000","last_peered":"2024-09-13T14:59:50.919420+0000","last_clean":"2024-09-13T14:59:50.919420+0000","last_became_active":"2024-09-13T14:43:35.906737+0000","last_became_peered":"2024-09-13T14:43:35.906737+0000","last_unstale":"2024-09-13T14:59:50.919420+0000","last_undegraded":"2024-09-13T14:59:50.919420+0000","last_fullsized":"2024-09-13T14:59:50.919420+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:04:32.577439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":308,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T14:59:59.959398+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T14:59:59.959398+0000","last_peered":"2024-09-13T14:59:59.959398+0000","last_clean":"2024-09-13T14:59:59.959398+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T14:59:59.959398+0000","last_undegraded":"2024-09-13T14:59:59.959398+0000","last_fullsized":"2024-09-13T14:59:59.959398+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.3","version":"335'9447","reported_seq":9587,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.008915+0000","last_change":"2024-09-13T14:58:00.208715+0000","last_active":"2024-09-13T15:00:07.008915+0000","last_peered":"2024-09-13T15:00:07.008915+0000","last_clean":"2024-09-13T14:57:59.112552+0000","last_became_active":"2024-09-13T14:58:00.208715+0000","last_became_peered":"2024-09-13T14:58:00.208715+0000","last_unstale":"2024-09-13T15:00:07.008915+0000","last_undegraded":"2024-09-13T14:58:00.197342+0000","last_fullsized":"2024-09-13T14:58:00.197242+0000","mapping_epoch":315,"log_start":"302'2348","ondisk_log_start":"302'2348","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5072","last_scrub_stamp":"2024-09-13T14:57:49.601547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.601547+0000","objects_scrubbed":2726,"log_size":7099,"log_dups_size":0,"ondisk_log_size":7099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:01.037695+0000","scrub_duration":891,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25427968,"num_objects":388,"num_object_clones":0,"num_object_copies":1164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":388,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4388,"num_write_kb":65600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":388}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919926+0000","last_change":"2024-09-13T14:40:37.182177+0000","last_active":"2024-09-13T14:59:50.919926+0000","last_peered":"2024-09-13T14:59:50.919926+0000","last_clean":"2024-09-13T14:59:50.919926+0000","last_became_active":"2024-09-13T14:40:37.181839+0000","last_became_peered":"2024-09-13T14:40:37.181839+0000","last_unstale":"2024-09-13T14:59:50.919926+0000","last_undegraded":"2024-09-13T14:59:50.919926+0000","last_fullsized":"2024-09-13T14:59:50.919926+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:12:44.081576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2024-09-13T15:00:08.006217+0000","last_change":"2024-09-13T14:59:13.689224+0000","last_active":"2024-09-13T15:00:08.006217+0000","last_peered":"2024-09-13T15:00:08.006217+0000","last_clean":"2024-09-13T15:00:08.006217+0000","last_became_active":"2024-09-13T14:59:13.688798+0000","last_became_peered":"2024-09-13T14:59:13.688798+0000","last_unstale":"2024-09-13T15:00:08.006217+0000","last_undegraded":"2024-09-13T15:00:08.006217+0000","last_fullsized":"2024-09-13T15:00:08.006217+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:54:51.804710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.4","version":"335'9315","reported_seq":9440,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.903190+0000","last_change":"2024-09-13T14:57:55.202026+0000","last_active":"2024-09-13T15:00:05.903190+0000","last_peered":"2024-09-13T15:00:05.903190+0000","last_clean":"2024-09-13T15:00:05.903190+0000","last_became_active":"2024-09-13T14:57:42.089977+0000","last_became_peered":"2024-09-13T14:57:42.089977+0000","last_unstale":"2024-09-13T15:00:05.903190+0000","last_undegraded":"2024-09-13T15:00:05.903190+0000","last_fullsized":"2024-09-13T15:00:05.903190+0000","mapping_epoch":298,"log_start":"301'881","ondisk_log_start":"301'881","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"314'5167","last_scrub_stamp":"2024-09-13T14:57:55.201860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:55.201860+0000","objects_scrubbed":2796,"log_size":8434,"log_dups_size":0,"ondisk_log_size":8434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:49:43.151657+0000","scrub_duration":0.69252925899999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25362432,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4159,"num_write_kb":56000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":848,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.005188+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:00:07.005188+0000","last_peered":"2024-09-13T15:00:07.005188+0000","last_clean":"2024-09-13T15:00:07.005188+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:00:07.005188+0000","last_undegraded":"2024-09-13T15:00:07.005188+0000","last_fullsized":"2024-09-13T15:00:07.005188+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":554,"reported_epoch":336,"state":"active+clean+remapped","last_fresh":"2024-09-13T15:00:07.008659+0000","last_change":"2024-09-13T14:59:13.698027+0000","last_active":"2024-09-13T15:00:07.008659+0000","last_peered":"2024-09-13T15:00:07.008659+0000","last_clean":"2024-09-13T15:00:07.008659+0000","last_became_active":"2024-09-13T14:59:13.689548+0000","last_became_peered":"2024-09-13T14:59:13.689548+0000","last_unstale":"2024-09-13T15:00:07.008659+0000","last_undegraded":"2024-09-13T15:00:07.008659+0000","last_fullsized":"2024-09-13T15:00:07.008659+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:52.358606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941410+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T14:59:50.941410+0000","last_peered":"2024-09-13T14:59:50.941410+0000","last_clean":"2024-09-13T14:59:50.941410+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T14:59:50.941410+0000","last_undegraded":"2024-09-13T14:59:50.941410+0000","last_fullsized":"2024-09-13T14:59:50.941410+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.5","version":"335'9210","reported_seq":9333,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.798867+0000","last_change":"2024-09-13T14:57:54.124864+0000","last_active":"2024-09-13T15:00:05.798867+0000","last_peered":"2024-09-13T15:00:05.798867+0000","last_clean":"2024-09-13T15:00:05.798867+0000","last_became_active":"2024-09-13T14:57:42.086633+0000","last_became_peered":"2024-09-13T14:57:42.086633+0000","last_unstale":"2024-09-13T15:00:05.798867+0000","last_undegraded":"2024-09-13T15:00:05.798867+0000","last_fullsized":"2024-09-13T15:00:05.798867+0000","mapping_epoch":298,"log_start":"301'614","ondisk_log_start":"301'614","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"314'5087","last_scrub_stamp":"2024-09-13T14:57:54.124643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:54.124643+0000","objects_scrubbed":2709,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:31:22.438092+0000","scrub_duration":0.60692724600000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26411008,"num_objects":403,"num_object_clones":0,"num_object_copies":1209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4128,"num_write_kb":58304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":576,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919182+0000","last_change":"2024-09-13T14:42:59.342067+0000","last_active":"2024-09-13T14:59:50.919182+0000","last_peered":"2024-09-13T14:59:50.919182+0000","last_clean":"2024-09-13T14:59:50.919182+0000","last_became_active":"2024-09-13T14:42:56.296368+0000","last_became_peered":"2024-09-13T14:42:56.296368+0000","last_unstale":"2024-09-13T14:59:50.919182+0000","last_undegraded":"2024-09-13T14:59:50.919182+0000","last_fullsized":"2024-09-13T14:59:50.919182+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:59.341925+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:59.341925+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:54:54.585363+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":356,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T14:59:59.959451+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T14:59:59.959451+0000","last_peered":"2024-09-13T14:59:59.959451+0000","last_clean":"2024-09-13T14:59:59.959451+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T14:59:59.959451+0000","last_undegraded":"2024-09-13T14:59:59.959451+0000","last_fullsized":"2024-09-13T14:59:59.959451+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941621+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T14:59:50.941621+0000","last_peered":"2024-09-13T14:59:50.941621+0000","last_clean":"2024-09-13T14:59:50.941621+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T14:59:50.941621+0000","last_undegraded":"2024-09-13T14:59:50.941621+0000","last_fullsized":"2024-09-13T14:59:50.941621+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.6","version":"337'9178","reported_seq":9317,"reported_epoch":337,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:09.715866+0000","last_change":"2024-09-13T14:58:00.209329+0000","last_active":"2024-09-13T15:00:09.715866+0000","last_peered":"2024-09-13T15:00:09.715866+0000","last_clean":"2024-09-13T14:57:59.167005+0000","last_became_active":"2024-09-13T14:58:00.209329+0000","last_became_peered":"2024-09-13T14:58:00.209329+0000","last_unstale":"2024-09-13T15:00:09.715866+0000","last_undegraded":"2024-09-13T14:58:00.192021+0000","last_fullsized":"2024-09-13T14:58:00.191919+0000","mapping_epoch":315,"log_start":"301'1111","ondisk_log_start":"301'1111","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5005","last_scrub_stamp":"2024-09-13T14:57:50.368944+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.368944+0000","objects_scrubbed":2603,"log_size":8067,"log_dups_size":0,"ondisk_log_size":8067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:17.407794+0000","scrub_duration":369,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18743296,"num_objects":286,"num_object_clones":0,"num_object_copies":858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":286,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":286,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4173,"num_write_kb":59392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":286}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940664+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T14:59:50.940664+0000","last_peered":"2024-09-13T14:59:50.940664+0000","last_clean":"2024-09-13T14:59:50.940664+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T14:59:50.940664+0000","last_undegraded":"2024-09-13T14:59:50.940664+0000","last_fullsized":"2024-09-13T14:59:50.940664+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.942082+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T14:59:50.942082+0000","last_peered":"2024-09-13T14:59:50.942082+0000","last_clean":"2024-09-13T14:59:50.942082+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T14:59:50.942082+0000","last_undegraded":"2024-09-13T14:59:50.942082+0000","last_fullsized":"2024-09-13T14:59:50.942082+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.7","version":"337'9327","reported_seq":9463,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:09.767297+0000","last_change":"2024-09-13T14:57:49.747386+0000","last_active":"2024-09-13T15:00:09.767297+0000","last_peered":"2024-09-13T15:00:09.767297+0000","last_clean":"2024-09-13T15:00:09.767297+0000","last_became_active":"2024-09-13T14:57:42.095484+0000","last_became_peered":"2024-09-13T14:57:42.095484+0000","last_unstale":"2024-09-13T15:00:09.767297+0000","last_undegraded":"2024-09-13T15:00:09.767297+0000","last_fullsized":"2024-09-13T15:00:09.767297+0000","mapping_epoch":298,"log_start":"301'1376","ondisk_log_start":"301'1376","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5022","last_scrub_stamp":"2024-09-13T14:57:49.747144+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.747144+0000","objects_scrubbed":2617,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:01.334229+0000","scrub_duration":717,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4314,"num_write_kb":63552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":576,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919339+0000","last_change":"2024-09-13T14:42:57.290955+0000","last_active":"2024-09-13T14:59:50.919339+0000","last_peered":"2024-09-13T14:59:50.919339+0000","last_clean":"2024-09-13T14:59:50.919339+0000","last_became_active":"2024-09-13T14:42:56.289675+0000","last_became_peered":"2024-09-13T14:42:56.289675+0000","last_unstale":"2024-09-13T14:59:50.919339+0000","last_undegraded":"2024-09-13T14:59:50.919339+0000","last_fullsized":"2024-09-13T14:59:50.919339+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.290663+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.290663+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:31:44.004222+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":500,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.007561+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:00:07.007561+0000","last_peered":"2024-09-13T15:00:07.007561+0000","last_clean":"2024-09-13T15:00:07.007561+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:00:07.007561+0000","last_undegraded":"2024-09-13T15:00:07.007561+0000","last_fullsized":"2024-09-13T15:00:07.007561+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":514,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006579+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:00:08.006579+0000","last_peered":"2024-09-13T15:00:08.006579+0000","last_clean":"2024-09-13T15:00:08.006579+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:00:08.006579+0000","last_undegraded":"2024-09-13T15:00:08.006579+0000","last_fullsized":"2024-09-13T15:00:08.006579+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.d","version":"337'13896","reported_seq":13992,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:09.783870+0000","last_change":"2024-09-13T14:56:44.021769+0000","last_active":"2024-09-13T15:00:09.783870+0000","last_peered":"2024-09-13T15:00:09.783870+0000","last_clean":"2024-09-13T15:00:09.783870+0000","last_became_active":"2024-09-13T14:56:44.021451+0000","last_became_peered":"2024-09-13T14:56:44.021451+0000","last_unstale":"2024-09-13T15:00:09.783870+0000","last_undegraded":"2024-09-13T15:00:09.783870+0000","last_fullsized":"2024-09-13T15:00:09.783870+0000","mapping_epoch":298,"log_start":"318'7600","ondisk_log_start":"318'7600","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:41:29.177584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43843608,"num_objects":670,"num_object_clones":0,"num_object_copies":2010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":670,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13896,"num_write_kb":466049,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":541,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-13T14:59:59.959491+0000","last_change":"2024-09-13T14:58:00.196065+0000","last_active":"2024-09-13T14:59:59.959491+0000","last_peered":"2024-09-13T14:59:59.959491+0000","last_clean":"2024-09-13T14:54:54.194954+0000","last_became_active":"2024-09-13T14:58:00.196065+0000","last_became_peered":"2024-09-13T14:58:00.196065+0000","last_unstale":"2024-09-13T14:59:59.959491+0000","last_undegraded":"2024-09-13T14:59:59.959491+0000","last_fullsized":"2024-09-13T14:58:00.191925+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:22:33.310023+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":322,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919449+0000","last_change":"2024-09-13T14:40:02.366911+0000","last_active":"2024-09-13T14:59:50.919449+0000","last_peered":"2024-09-13T14:59:50.919449+0000","last_clean":"2024-09-13T14:59:50.919449+0000","last_became_active":"2024-09-13T14:40:02.366473+0000","last_became_peered":"2024-09-13T14:40:02.366473+0000","last_unstale":"2024-09-13T14:59:50.919449+0000","last_undegraded":"2024-09-13T14:59:50.919449+0000","last_fullsized":"2024-09-13T14:59:50.919449+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:56.875262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":485,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.011023+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:00:07.011023+0000","last_peered":"2024-09-13T15:00:07.011023+0000","last_clean":"2024-09-13T15:00:07.011023+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:00:07.011023+0000","last_undegraded":"2024-09-13T15:00:07.011023+0000","last_fullsized":"2024-09-13T15:00:07.011023+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":317,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.928282+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T14:59:50.928282+0000","last_peered":"2024-09-13T14:59:50.928282+0000","last_clean":"2024-09-13T14:59:50.928282+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T14:59:50.928282+0000","last_undegraded":"2024-09-13T14:59:50.928282+0000","last_fullsized":"2024-09-13T14:59:50.928282+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:59:13.033398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"12.e","version":"335'13781","reported_seq":13878,"reported_epoch":336,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-09-13T15:00:07.005069+0000","last_change":"2024-09-13T14:59:13.842972+0000","last_active":"2024-09-13T15:00:07.005069+0000","last_peered":"2024-09-13T15:00:07.005069+0000","last_clean":"2024-09-13T14:57:59.154960+0000","last_became_active":"2024-09-13T14:59:13.815667+0000","last_became_peered":"2024-09-13T14:59:13.815667+0000","last_unstale":"2024-09-13T15:00:07.005069+0000","last_undegraded":"2024-09-13T14:59:13.701823+0000","last_fullsized":"2024-09-13T14:59:13.684281+0000","mapping_epoch":326,"log_start":"318'7871","ondisk_log_start":"318'7871","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47382528,"num_objects":723,"num_object_clones":0,"num_object_copies":2169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":723,"num_objects_unfound":0,"num_objects_dirty":723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13781,"num_write_kb":464128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":723}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":782,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006105+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:00:08.006105+0000","last_peered":"2024-09-13T15:00:08.006105+0000","last_clean":"2024-09-13T15:00:08.006105+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:00:08.006105+0000","last_undegraded":"2024-09-13T15:00:08.006105+0000","last_fullsized":"2024-09-13T15:00:08.006105+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":492,"reported_epoch":337,"state":"active+undersized","last_fresh":"2024-09-13T15:00:08.005761+0000","last_change":"2024-09-13T14:58:00.196079+0000","last_active":"2024-09-13T15:00:08.005761+0000","last_peered":"2024-09-13T15:00:08.005761+0000","last_clean":"2024-09-13T14:57:53.147377+0000","last_became_active":"2024-09-13T14:58:00.196079+0000","last_became_peered":"2024-09-13T14:58:00.196079+0000","last_unstale":"2024-09-13T15:00:08.005761+0000","last_undegraded":"2024-09-13T15:00:08.005761+0000","last_fullsized":"2024-09-13T14:58:00.192042+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:50.213407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.940402+0000","last_change":"2024-09-13T14:59:13.720055+0000","last_active":"2024-09-13T14:59:50.940402+0000","last_peered":"2024-09-13T14:59:50.940402+0000","last_clean":"2024-09-13T14:59:50.940402+0000","last_became_active":"2024-09-13T14:59:13.719639+0000","last_became_peered":"2024-09-13T14:59:13.719639+0000","last_unstale":"2024-09-13T14:59:50.940402+0000","last_undegraded":"2024-09-13T14:59:50.940402+0000","last_fullsized":"2024-09-13T14:59:50.940402+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:48.915754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,5],"avail_no_missing":["4","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":382,"reported_epoch":337,"state":"active+undersized","last_fresh":"2024-09-13T15:00:08.006477+0000","last_change":"2024-09-13T14:58:00.197201+0000","last_active":"2024-09-13T15:00:08.006477+0000","last_peered":"2024-09-13T15:00:08.006477+0000","last_clean":"2024-09-13T14:54:54.205156+0000","last_became_active":"2024-09-13T14:58:00.197201+0000","last_became_peered":"2024-09-13T14:58:00.197201+0000","last_unstale":"2024-09-13T15:00:08.006477+0000","last_undegraded":"2024-09-13T15:00:08.006477+0000","last_fullsized":"2024-09-13T14:58:00.190432+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:55:48.403571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.f","version":"335'13697","reported_seq":13744,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.894053+0000","last_change":"2024-09-13T14:56:43.814995+0000","last_active":"2024-09-13T15:00:05.894053+0000","last_peered":"2024-09-13T15:00:05.894053+0000","last_clean":"2024-09-13T15:00:05.894053+0000","last_became_active":"2024-09-13T14:56:43.814719+0000","last_became_peered":"2024-09-13T14:56:43.814719+0000","last_unstale":"2024-09-13T15:00:05.894053+0000","last_undegraded":"2024-09-13T15:00:05.894053+0000","last_fullsized":"2024-09-13T15:00:05.894053+0000","mapping_epoch":298,"log_start":"318'7100","ondisk_log_start":"318'7100","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.360102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13697,"num_write_kb":465344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":587,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.941336+0000","last_change":"2024-09-13T14:59:15.701141+0000","last_active":"2024-09-13T14:59:50.941336+0000","last_peered":"2024-09-13T14:59:50.941336+0000","last_clean":"2024-09-13T14:59:50.941336+0000","last_became_active":"2024-09-13T14:59:15.700696+0000","last_became_peered":"2024-09-13T14:59:15.700696+0000","last_unstale":"2024-09-13T14:59:50.941336+0000","last_undegraded":"2024-09-13T14:59:50.941336+0000","last_fullsized":"2024-09-13T14:59:50.941336+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:01:36.489123+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":541,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.003745+0000","last_change":"2024-09-13T14:58:00.209707+0000","last_active":"2024-09-13T15:00:07.003745+0000","last_peered":"2024-09-13T15:00:07.003745+0000","last_clean":"2024-09-13T14:57:53.147953+0000","last_became_active":"2024-09-13T14:58:00.209707+0000","last_became_peered":"2024-09-13T14:58:00.209707+0000","last_unstale":"2024-09-13T15:00:07.003745+0000","last_undegraded":"2024-09-13T15:00:07.003745+0000","last_fullsized":"2024-09-13T14:58:00.188724+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:44.693424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":500,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006032+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:00:08.006032+0000","last_peered":"2024-09-13T15:00:08.006032+0000","last_clean":"2024-09-13T15:00:08.006032+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:00:08.006032+0000","last_undegraded":"2024-09-13T15:00:08.006032+0000","last_fullsized":"2024-09-13T15:00:08.006032+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":554,"reported_epoch":337,"state":"active+undersized","last_fresh":"2024-09-13T15:00:08.006372+0000","last_change":"2024-09-13T14:58:00.196078+0000","last_active":"2024-09-13T15:00:08.006372+0000","last_peered":"2024-09-13T15:00:08.006372+0000","last_clean":"2024-09-13T14:57:53.149012+0000","last_became_active":"2024-09-13T14:58:00.196078+0000","last_became_peered":"2024-09-13T14:58:00.196078+0000","last_unstale":"2024-09-13T15:00:08.006372+0000","last_undegraded":"2024-09-13T15:00:08.006372+0000","last_fullsized":"2024-09-13T14:58:00.192052+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:05:57.351796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.c","version":"335'13556","reported_seq":13629,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:05.903371+0000","last_change":"2024-09-13T14:58:00.200526+0000","last_active":"2024-09-13T15:00:05.903371+0000","last_peered":"2024-09-13T15:00:05.903371+0000","last_clean":"2024-09-13T14:57:59.094224+0000","last_became_active":"2024-09-13T14:58:00.200526+0000","last_became_peered":"2024-09-13T14:58:00.200526+0000","last_unstale":"2024-09-13T15:00:05.903371+0000","last_undegraded":"2024-09-13T14:58:00.190108+0000","last_fullsized":"2024-09-13T14:58:00.189997+0000","mapping_epoch":315,"log_start":"305'4400","ondisk_log_start":"305'4400","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":9156,"log_dups_size":0,"ondisk_log_size":9156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:03:50.813569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":832,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13556,"num_write_kb":460416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":832}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":617,"reported_epoch":334,"state":"active+undersized","last_fresh":"2024-09-13T14:59:50.920908+0000","last_change":"2024-09-13T14:58:00.206124+0000","last_active":"2024-09-13T14:59:50.920908+0000","last_peered":"2024-09-13T14:59:50.920908+0000","last_clean":"2024-09-13T14:57:53.396018+0000","last_became_active":"2024-09-13T14:58:00.206124+0000","last_became_peered":"2024-09-13T14:58:00.206124+0000","last_unstale":"2024-09-13T14:59:50.920908+0000","last_undegraded":"2024-09-13T14:59:50.920908+0000","last_fullsized":"2024-09-13T14:58:00.189988+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:24.008447+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":323,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919426+0000","last_change":"2024-09-13T14:35:58.860267+0000","last_active":"2024-09-13T14:59:50.919426+0000","last_peered":"2024-09-13T14:59:50.919426+0000","last_clean":"2024-09-13T14:59:50.919426+0000","last_became_active":"2024-09-13T14:35:58.859351+0000","last_became_peered":"2024-09-13T14:35:58.859351+0000","last_unstale":"2024-09-13T14:59:50.919426+0000","last_undegraded":"2024-09-13T14:59:50.919426+0000","last_fullsized":"2024-09-13T14:59:50.919426+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:21:25.762769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.940975+0000","last_change":"2024-09-13T14:59:13.687964+0000","last_active":"2024-09-13T14:59:50.940975+0000","last_peered":"2024-09-13T14:59:50.940975+0000","last_clean":"2024-09-13T14:59:50.940975+0000","last_became_active":"2024-09-13T14:59:13.687546+0000","last_became_peered":"2024-09-13T14:59:13.687546+0000","last_unstale":"2024-09-13T14:59:50.940975+0000","last_undegraded":"2024-09-13T14:59:50.940975+0000","last_fullsized":"2024-09-13T14:59:50.940975+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:11:22.830176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":512,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006185+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:00:08.006185+0000","last_peered":"2024-09-13T15:00:08.006185+0000","last_clean":"2024-09-13T15:00:08.006185+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:00:08.006185+0000","last_undegraded":"2024-09-13T15:00:08.006185+0000","last_fullsized":"2024-09-13T15:00:08.006185+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.a","version":"336'13936","reported_seq":14030,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.370114+0000","last_change":"2024-09-13T14:58:00.209181+0000","last_active":"2024-09-13T15:00:07.370114+0000","last_peered":"2024-09-13T15:00:07.370114+0000","last_clean":"2024-09-13T14:57:59.161016+0000","last_became_active":"2024-09-13T14:58:00.209181+0000","last_became_peered":"2024-09-13T14:58:00.209181+0000","last_unstale":"2024-09-13T15:00:07.370114+0000","last_undegraded":"2024-09-13T14:58:00.188435+0000","last_fullsized":"2024-09-13T14:58:00.188304+0000","mapping_epoch":315,"log_start":"318'7100","ondisk_log_start":"318'7100","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:50.670496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47579136,"num_objects":726,"num_object_clones":0,"num_object_copies":2178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":726,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13936,"num_write_kb":469184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"2,3","objects":726}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":584,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.928450+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T14:59:50.928450+0000","last_peered":"2024-09-13T14:59:50.928450+0000","last_clean":"2024-09-13T14:59:50.928450+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T14:59:50.928450+0000","last_undegraded":"2024-09-13T14:59:50.928450+0000","last_fullsized":"2024-09-13T14:59:50.928450+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":323,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919711+0000","last_change":"2024-09-13T14:40:37.182171+0000","last_active":"2024-09-13T14:59:50.919711+0000","last_peered":"2024-09-13T14:59:50.919711+0000","last_clean":"2024-09-13T14:59:50.919711+0000","last_became_active":"2024-09-13T14:40:37.181867+0000","last_became_peered":"2024-09-13T14:40:37.181867+0000","last_unstale":"2024-09-13T14:59:50.919711+0000","last_undegraded":"2024-09-13T14:59:50.919711+0000","last_fullsized":"2024-09-13T14:59:50.919711+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:13.678955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":568,"reported_epoch":337,"state":"active+undersized","last_fresh":"2024-09-13T15:00:08.006439+0000","last_change":"2024-09-13T14:58:00.198485+0000","last_active":"2024-09-13T15:00:08.006439+0000","last_peered":"2024-09-13T15:00:08.006439+0000","last_clean":"2024-09-13T14:57:53.148529+0000","last_became_active":"2024-09-13T14:58:00.198485+0000","last_became_peered":"2024-09-13T14:58:00.198485+0000","last_unstale":"2024-09-13T15:00:08.006439+0000","last_undegraded":"2024-09-13T15:00:08.006439+0000","last_fullsized":"2024-09-13T14:58:00.194105+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:25.937883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":439,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.005924+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:00:08.005924+0000","last_peered":"2024-09-13T15:00:08.005924+0000","last_clean":"2024-09-13T15:00:08.005924+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:00:08.005924+0000","last_undegraded":"2024-09-13T15:00:08.005924+0000","last_fullsized":"2024-09-13T15:00:08.005924+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.b","version":"335'13816","reported_seq":13908,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.008599+0000","last_change":"2024-09-13T14:56:43.815601+0000","last_active":"2024-09-13T15:00:07.008599+0000","last_peered":"2024-09-13T15:00:07.008599+0000","last_clean":"2024-09-13T15:00:07.008599+0000","last_became_active":"2024-09-13T14:56:43.815322+0000","last_became_peered":"2024-09-13T14:56:43.815322+0000","last_unstale":"2024-09-13T15:00:07.008599+0000","last_undegraded":"2024-09-13T15:00:07.008599+0000","last_fullsized":"2024-09-13T15:00:07.008599+0000","mapping_epoch":298,"log_start":"318'7900","ondisk_log_start":"318'7900","created":298,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":5916,"log_dups_size":0,"ondisk_log_size":5916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:30:49.708244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51380224,"num_objects":784,"num_object_clones":0,"num_object_copies":2352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13816,"num_write_kb":467200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":624,"reported_epoch":334,"state":"active+undersized","last_fresh":"2024-09-13T14:59:50.928291+0000","last_change":"2024-09-13T14:58:00.211752+0000","last_active":"2024-09-13T14:59:50.928291+0000","last_peered":"2024-09-13T14:59:50.928291+0000","last_clean":"2024-09-13T14:57:53.239177+0000","last_became_active":"2024-09-13T14:58:00.211752+0000","last_became_peered":"2024-09-13T14:58:00.211752+0000","last_unstale":"2024-09-13T14:59:50.928291+0000","last_undegraded":"2024-09-13T14:59:50.928291+0000","last_fullsized":"2024-09-13T14:58:00.191604+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:25.585853+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":499,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006327+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:00:08.006327+0000","last_peered":"2024-09-13T15:00:08.006327+0000","last_clean":"2024-09-13T15:00:08.006327+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:00:08.006327+0000","last_undegraded":"2024-09-13T15:00:08.006327+0000","last_fullsized":"2024-09-13T15:00:08.006327+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.941866+0000","last_change":"2024-09-13T14:59:13.687794+0000","last_active":"2024-09-13T14:59:50.941866+0000","last_peered":"2024-09-13T14:59:50.941866+0000","last_clean":"2024-09-13T14:59:50.941866+0000","last_became_active":"2024-09-13T14:59:13.687365+0000","last_became_peered":"2024-09-13T14:59:13.687365+0000","last_unstale":"2024-09-13T14:59:50.941866+0000","last_undegraded":"2024-09-13T14:59:50.941866+0000","last_fullsized":"2024-09-13T14:59:50.941866+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:26:34.937515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":335,"state":"active+undersized","last_fresh":"2024-09-13T14:59:59.963064+0000","last_change":"2024-09-13T14:58:00.196530+0000","last_active":"2024-09-13T14:59:59.963064+0000","last_peered":"2024-09-13T14:59:59.963064+0000","last_clean":"2024-09-13T14:20:45.753839+0000","last_became_active":"2024-09-13T14:58:00.196530+0000","last_became_peered":"2024-09-13T14:58:00.196530+0000","last_unstale":"2024-09-13T14:59:59.963064+0000","last_undegraded":"2024-09-13T14:59:59.963064+0000","last_fullsized":"2024-09-13T14:58:00.192715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:08:52.804801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.8","version":"335'9360","reported_seq":9500,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.007008+0000","last_change":"2024-09-13T14:58:00.267572+0000","last_active":"2024-09-13T15:00:07.007008+0000","last_peered":"2024-09-13T15:00:07.007008+0000","last_clean":"2024-09-13T14:57:59.131526+0000","last_became_active":"2024-09-13T14:58:00.267572+0000","last_became_peered":"2024-09-13T14:58:00.267572+0000","last_unstale":"2024-09-13T15:00:07.007008+0000","last_undegraded":"2024-09-13T14:58:00.194520+0000","last_fullsized":"2024-09-13T14:58:00.194409+0000","mapping_epoch":315,"log_start":"302'2128","ondisk_log_start":"302'2128","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5084","last_scrub_stamp":"2024-09-13T14:57:50.981843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.981843+0000","objects_scrubbed":2677,"log_size":7232,"log_dups_size":0,"ondisk_log_size":7232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:11.598922+0000","scrub_duration":264,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26017792,"num_objects":397,"num_object_clones":0,"num_object_copies":1191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":397,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":63872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":397}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":513,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941782+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T14:59:50.941782+0000","last_peered":"2024-09-13T14:59:50.941782+0000","last_clean":"2024-09-13T14:59:50.941782+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T14:59:50.941782+0000","last_undegraded":"2024-09-13T14:59:50.941782+0000","last_fullsized":"2024-09-13T14:59:50.941782+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":512,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.004922+0000","last_change":"2024-09-13T14:58:00.206311+0000","last_active":"2024-09-13T15:00:07.004922+0000","last_peered":"2024-09-13T15:00:07.004922+0000","last_clean":"2024-09-13T14:57:56.555019+0000","last_became_active":"2024-09-13T14:58:00.206311+0000","last_became_peered":"2024-09-13T14:58:00.206311+0000","last_unstale":"2024-09-13T15:00:07.004922+0000","last_undegraded":"2024-09-13T14:58:00.194308+0000","last_fullsized":"2024-09-13T14:58:00.193842+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:05.897239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":514,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006681+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:00:08.006681+0000","last_peered":"2024-09-13T15:00:08.006681+0000","last_clean":"2024-09-13T15:00:08.006681+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:00:08.006681+0000","last_undegraded":"2024-09-13T15:00:08.006681+0000","last_fullsized":"2024-09-13T15:00:08.006681+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":515,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.007188+0000","last_change":"2024-09-13T14:58:00.200367+0000","last_active":"2024-09-13T15:00:07.007188+0000","last_peered":"2024-09-13T15:00:07.007188+0000","last_clean":"2024-09-13T14:57:53.144224+0000","last_became_active":"2024-09-13T14:58:00.200367+0000","last_became_peered":"2024-09-13T14:58:00.200367+0000","last_unstale":"2024-09-13T15:00:07.007188+0000","last_undegraded":"2024-09-13T15:00:07.007188+0000","last_fullsized":"2024-09-13T14:58:00.192544+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:08.459840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.9","version":"336'9433","reported_seq":9573,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.340789+0000","last_change":"2024-09-13T14:57:50.265028+0000","last_active":"2024-09-13T15:00:07.340789+0000","last_peered":"2024-09-13T15:00:07.340789+0000","last_clean":"2024-09-13T15:00:07.340789+0000","last_became_active":"2024-09-13T14:57:42.096368+0000","last_became_peered":"2024-09-13T14:57:42.096368+0000","last_unstale":"2024-09-13T15:00:07.340789+0000","last_undegraded":"2024-09-13T15:00:07.340789+0000","last_fullsized":"2024-09-13T15:00:07.340789+0000","mapping_epoch":298,"log_start":"301'463","ondisk_log_start":"301'463","created":298,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"310'5129","last_scrub_stamp":"2024-09-13T14:57:50.264920+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.264920+0000","objects_scrubbed":2737,"log_size":8970,"log_dups_size":0,"ondisk_log_size":8970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:41:16.799883+0000","scrub_duration":521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24838144,"num_objects":379,"num_object_clones":0,"num_object_copies":1137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4304,"num_write_kb":62272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":759,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.004411+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:00:07.004411+0000","last_peered":"2024-09-13T15:00:07.004411+0000","last_clean":"2024-09-13T15:00:07.004411+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:00:07.004411+0000","last_undegraded":"2024-09-13T15:00:07.004411+0000","last_fullsized":"2024-09-13T15:00:07.004411+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":330,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2024-09-13T15:00:08.006513+0000","last_change":"2024-09-13T14:59:13.691543+0000","last_active":"2024-09-13T15:00:08.006513+0000","last_peered":"2024-09-13T15:00:08.006513+0000","last_clean":"2024-09-13T15:00:08.006513+0000","last_became_active":"2024-09-13T14:59:13.687162+0000","last_became_peered":"2024-09-13T14:59:13.687162+0000","last_unstale":"2024-09-13T15:00:08.006513+0000","last_undegraded":"2024-09-13T15:00:08.006513+0000","last_fullsized":"2024-09-13T15:00:08.006513+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:32.615151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,7],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":321,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.928494+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T14:59:50.928494+0000","last_peered":"2024-09-13T14:59:50.928494+0000","last_clean":"2024-09-13T14:59:50.928494+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T14:59:50.928494+0000","last_undegraded":"2024-09-13T14:59:50.928494+0000","last_fullsized":"2024-09-13T14:59:50.928494+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940526+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T14:59:50.940526+0000","last_peered":"2024-09-13T14:59:50.940526+0000","last_clean":"2024-09-13T14:59:50.940526+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T14:59:50.940526+0000","last_undegraded":"2024-09-13T14:59:50.940526+0000","last_fullsized":"2024-09-13T14:59:50.940526+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:41.734273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941093+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T14:59:50.941093+0000","last_peered":"2024-09-13T14:59:50.941093+0000","last_clean":"2024-09-13T14:59:50.941093+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T14:59:50.941093+0000","last_undegraded":"2024-09-13T14:59:50.941093+0000","last_fullsized":"2024-09-13T14:59:50.941093+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":30,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941336+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T14:59:50.941336+0000","last_peered":"2024-09-13T14:59:50.941336+0000","last_clean":"2024-09-13T14:59:50.941336+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T14:59:50.941336+0000","last_undegraded":"2024-09-13T14:59:50.941336+0000","last_fullsized":"2024-09-13T14:59:50.941336+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":420,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.006062+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:00:08.006062+0000","last_peered":"2024-09-13T15:00:08.006062+0000","last_clean":"2024-09-13T15:00:08.006062+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:00:08.006062+0000","last_undegraded":"2024-09-13T15:00:08.006062+0000","last_fullsized":"2024-09-13T15:00:08.006062+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":473,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.007648+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:00:07.007648+0000","last_peered":"2024-09-13T15:00:07.007648+0000","last_clean":"2024-09-13T15:00:07.007648+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:00:07.007648+0000","last_undegraded":"2024-09-13T15:00:07.007648+0000","last_fullsized":"2024-09-13T15:00:07.007648+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"12.19","version":"336'9387","reported_seq":9527,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.388517+0000","last_change":"2024-09-13T14:57:49.565228+0000","last_active":"2024-09-13T15:00:07.388517+0000","last_peered":"2024-09-13T15:00:07.388517+0000","last_clean":"2024-09-13T15:00:07.388517+0000","last_became_active":"2024-09-13T14:57:42.106480+0000","last_became_peered":"2024-09-13T14:57:42.106480+0000","last_unstale":"2024-09-13T15:00:07.388517+0000","last_undegraded":"2024-09-13T15:00:07.388517+0000","last_fullsized":"2024-09-13T15:00:07.388517+0000","mapping_epoch":307,"log_start":"301'328","ondisk_log_start":"301'328","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5099","last_scrub_stamp":"2024-09-13T14:57:49.565078+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:49.565078+0000","objects_scrubbed":2693,"log_size":9059,"log_dups_size":0,"ondisk_log_size":9059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:13:01.042206+0000","scrub_duration":773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4295,"num_write_kb":63232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919796+0000","last_change":"2024-09-13T14:35:58.859283+0000","last_active":"2024-09-13T14:59:50.919796+0000","last_peered":"2024-09-13T14:59:50.919796+0000","last_clean":"2024-09-13T14:59:50.919796+0000","last_became_active":"2024-09-13T14:35:58.857633+0000","last_became_peered":"2024-09-13T14:35:58.857633+0000","last_unstale":"2024-09-13T14:59:50.919796+0000","last_undegraded":"2024-09-13T14:59:50.919796+0000","last_fullsized":"2024-09-13T14:59:50.919796+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:32.770667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":43,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.920815+0000","last_change":"2024-09-13T14:57:23.932331+0000","last_active":"2024-09-13T14:59:50.920815+0000","last_peered":"2024-09-13T14:59:50.920815+0000","last_clean":"2024-09-13T14:59:50.920815+0000","last_became_active":"2024-09-13T14:57:23.932039+0000","last_became_peered":"2024-09-13T14:57:23.932039+0000","last_unstale":"2024-09-13T14:59:50.920815+0000","last_undegraded":"2024-09-13T14:59:50.920815+0000","last_fullsized":"2024-09-13T14:59:50.920815+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:57:40.565036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.18","version":"335'9455","reported_seq":9596,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.006828+0000","last_change":"2024-09-13T14:58:00.266091+0000","last_active":"2024-09-13T15:00:07.006828+0000","last_peered":"2024-09-13T15:00:07.006828+0000","last_clean":"2024-09-13T14:57:59.225324+0000","last_became_active":"2024-09-13T14:58:00.266091+0000","last_became_peered":"2024-09-13T14:58:00.266091+0000","last_unstale":"2024-09-13T15:00:07.006828+0000","last_undegraded":"2024-09-13T14:58:00.194448+0000","last_fullsized":"2024-09-13T14:58:00.194336+0000","mapping_epoch":315,"log_start":"302'2429","ondisk_log_start":"302'2429","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5122","last_scrub_stamp":"2024-09-13T14:57:50.262095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.262095+0000","objects_scrubbed":2756,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:32:36.283129+0000","scrub_duration":540,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25296896,"num_objects":386,"num_object_clones":0,"num_object_copies":1158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":386,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4334,"num_write_kb":62848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":386}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":214,"reported_epoch":334,"state":"active+undersized","last_fresh":"2024-09-13T14:59:50.919821+0000","last_change":"2024-09-13T14:58:00.208196+0000","last_active":"2024-09-13T14:59:50.919821+0000","last_peered":"2024-09-13T14:59:50.919821+0000","last_clean":"2024-09-13T14:40:02.364287+0000","last_became_active":"2024-09-13T14:58:00.208196+0000","last_became_peered":"2024-09-13T14:58:00.208196+0000","last_unstale":"2024-09-13T14:59:50.919821+0000","last_undegraded":"2024-09-13T14:59:50.919821+0000","last_fullsized":"2024-09-13T14:58:00.193143+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:48:29.409728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":497,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.006926+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:00:07.006926+0000","last_peered":"2024-09-13T15:00:07.006926+0000","last_clean":"2024-09-13T15:00:07.006926+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:00:07.006926+0000","last_undegraded":"2024-09-13T15:00:07.006926+0000","last_fullsized":"2024-09-13T15:00:07.006926+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.17","version":"337'9302","reported_seq":9442,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:09.710717+0000","last_change":"2024-09-13T14:57:51.357626+0000","last_active":"2024-09-13T15:00:09.710717+0000","last_peered":"2024-09-13T15:00:09.710717+0000","last_clean":"2024-09-13T15:00:09.710717+0000","last_became_active":"2024-09-13T14:57:42.107486+0000","last_became_peered":"2024-09-13T14:57:42.107486+0000","last_unstale":"2024-09-13T15:00:09.710717+0000","last_undegraded":"2024-09-13T15:00:09.710717+0000","last_fullsized":"2024-09-13T15:00:09.710717+0000","mapping_epoch":307,"log_start":"302'1492","ondisk_log_start":"302'1492","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5002","last_scrub_stamp":"2024-09-13T14:57:51.357494+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:51.357494+0000","objects_scrubbed":2703,"log_size":7810,"log_dups_size":0,"ondisk_log_size":7810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:30:02.249380+0000","scrub_duration":322,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20250624,"num_objects":309,"num_object_clones":0,"num_object_copies":927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4300,"num_write_kb":60992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":308,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.928438+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T14:59:50.928438+0000","last_peered":"2024-09-13T14:59:50.928438+0000","last_clean":"2024-09-13T14:59:50.928438+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T14:59:50.928438+0000","last_undegraded":"2024-09-13T14:59:50.928438+0000","last_fullsized":"2024-09-13T14:59:50.928438+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":545,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.007352+0000","last_change":"2024-09-13T14:58:00.216756+0000","last_active":"2024-09-13T15:00:07.007352+0000","last_peered":"2024-09-13T15:00:07.007352+0000","last_clean":"2024-09-13T14:57:53.144315+0000","last_became_active":"2024-09-13T14:58:00.216756+0000","last_became_peered":"2024-09-13T14:58:00.216756+0000","last_unstale":"2024-09-13T15:00:07.007352+0000","last_undegraded":"2024-09-13T15:00:07.007352+0000","last_fullsized":"2024-09-13T14:58:00.198679+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:29:23.269572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.16","version":"337'9385","reported_seq":9529,"reported_epoch":337,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:09.790459+0000","last_change":"2024-09-13T14:58:00.198384+0000","last_active":"2024-09-13T15:00:09.790459+0000","last_peered":"2024-09-13T15:00:09.790459+0000","last_clean":"2024-09-13T14:57:59.150610+0000","last_became_active":"2024-09-13T14:58:00.198384+0000","last_became_peered":"2024-09-13T14:58:00.198384+0000","last_unstale":"2024-09-13T15:00:09.790459+0000","last_undegraded":"2024-09-13T14:58:00.192532+0000","last_fullsized":"2024-09-13T14:58:00.192417+0000","mapping_epoch":315,"log_start":"302'1586","ondisk_log_start":"302'1586","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"313'5055","last_scrub_stamp":"2024-09-13T14:57:52.447068+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:52.447068+0000","objects_scrubbed":2757,"log_size":7799,"log_dups_size":0,"ondisk_log_size":7799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:25.929132+0000","scrub_duration":434,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19333120,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4332,"num_write_kb":59840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":295}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":309,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.005405+0000","last_change":"2024-09-13T14:58:00.207661+0000","last_active":"2024-09-13T15:00:07.005405+0000","last_peered":"2024-09-13T15:00:07.005405+0000","last_clean":"2024-09-13T14:40:02.364918+0000","last_became_active":"2024-09-13T14:58:00.207661+0000","last_became_peered":"2024-09-13T14:58:00.207661+0000","last_unstale":"2024-09-13T15:00:07.005405+0000","last_undegraded":"2024-09-13T15:00:07.005405+0000","last_fullsized":"2024-09-13T14:58:00.194016+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:08:37.984416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":512,"reported_epoch":337,"state":"active+clean","last_fresh":"2024-09-13T15:00:08.005826+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:00:08.005826+0000","last_peered":"2024-09-13T15:00:08.005826+0000","last_clean":"2024-09-13T15:00:08.005826+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:00:08.005826+0000","last_undegraded":"2024-09-13T15:00:08.005826+0000","last_fullsized":"2024-09-13T15:00:08.005826+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"12.15","version":"335'9313","reported_seq":9444,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.884747+0000","last_change":"2024-09-13T14:58:08.102921+0000","last_active":"2024-09-13T15:00:05.884747+0000","last_peered":"2024-09-13T15:00:05.884747+0000","last_clean":"2024-09-13T15:00:05.884747+0000","last_became_active":"2024-09-13T14:57:42.092386+0000","last_became_peered":"2024-09-13T14:57:42.092386+0000","last_unstale":"2024-09-13T15:00:05.884747+0000","last_undegraded":"2024-09-13T15:00:05.884747+0000","last_fullsized":"2024-09-13T15:00:05.884747+0000","mapping_epoch":307,"log_start":"301'863","ondisk_log_start":"301'863","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"317'5489","last_scrub_stamp":"2024-09-13T14:58:08.102772+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.102772+0000","objects_scrubbed":3113,"log_size":8450,"log_dups_size":0,"ondisk_log_size":8450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:05.041889+0000","scrub_duration":0.68467245300000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30277632,"num_objects":462,"num_object_clones":0,"num_object_copies":1386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3833,"num_write_kb":37824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":510,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.007238+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:00:07.007238+0000","last_peered":"2024-09-13T15:00:07.007238+0000","last_clean":"2024-09-13T15:00:07.007238+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:00:07.007238+0000","last_undegraded":"2024-09-13T15:00:07.007238+0000","last_fullsized":"2024-09-13T15:00:07.007238+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.941246+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T14:59:50.941246+0000","last_peered":"2024-09-13T14:59:50.941246+0000","last_clean":"2024-09-13T14:59:50.941246+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T14:59:50.941246+0000","last_undegraded":"2024-09-13T14:59:50.941246+0000","last_fullsized":"2024-09-13T14:59:50.941246+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:17:39.699281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.14","version":"335'9238","reported_seq":9368,"reported_epoch":335,"state":"active+clean","last_fresh":"2024-09-13T15:00:05.915880+0000","last_change":"2024-09-13T14:58:08.964003+0000","last_active":"2024-09-13T15:00:05.915880+0000","last_peered":"2024-09-13T15:00:05.915880+0000","last_clean":"2024-09-13T15:00:05.915880+0000","last_became_active":"2024-09-13T14:57:42.094348+0000","last_became_peered":"2024-09-13T14:57:42.094348+0000","last_unstale":"2024-09-13T15:00:05.915880+0000","last_undegraded":"2024-09-13T15:00:05.915880+0000","last_fullsized":"2024-09-13T15:00:05.915880+0000","mapping_epoch":307,"log_start":"301'722","ondisk_log_start":"301'722","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"317'5506","last_scrub_stamp":"2024-09-13T14:58:08.963824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:58:08.963824+0000","objects_scrubbed":3147,"log_size":8516,"log_dups_size":0,"ondisk_log_size":8516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:21.311000+0000","scrub_duration":0.58457265300000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3744,"num_write_kb":32320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":523,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.002145+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:00:07.002145+0000","last_peered":"2024-09-13T15:00:07.002145+0000","last_clean":"2024-09-13T15:00:07.002145+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:00:07.002145+0000","last_undegraded":"2024-09-13T15:00:07.002145+0000","last_fullsized":"2024-09-13T15:00:07.002145+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":21,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.940836+0000","last_change":"2024-09-13T14:59:13.688428+0000","last_active":"2024-09-13T14:59:50.940836+0000","last_peered":"2024-09-13T14:59:50.940836+0000","last_clean":"2024-09-13T14:59:50.940836+0000","last_became_active":"2024-09-13T14:59:13.687721+0000","last_became_peered":"2024-09-13T14:59:13.687721+0000","last_unstale":"2024-09-13T14:59:50.940836+0000","last_undegraded":"2024-09-13T14:59:50.940836+0000","last_fullsized":"2024-09-13T14:59:50.940836+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:52:31.018560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,1],"avail_no_missing":["4","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"12.13","version":"336'9522","reported_seq":9662,"reported_epoch":336,"state":"active+undersized+degraded","last_fresh":"2024-09-13T15:00:07.008443+0000","last_change":"2024-09-13T14:58:00.210800+0000","last_active":"2024-09-13T15:00:07.008443+0000","last_peered":"2024-09-13T15:00:07.008443+0000","last_clean":"2024-09-13T14:57:59.137884+0000","last_became_active":"2024-09-13T14:58:00.210800+0000","last_became_peered":"2024-09-13T14:58:00.210800+0000","last_unstale":"2024-09-13T15:00:07.008443+0000","last_undegraded":"2024-09-13T14:58:00.197608+0000","last_fullsized":"2024-09-13T14:58:00.197497+0000","mapping_epoch":315,"log_start":"302'2503","ondisk_log_start":"302'2503","created":307,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":5,"last_scrub":"310'5142","last_scrub_stamp":"2024-09-13T14:57:50.224834+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:50.224834+0000","objects_scrubbed":2694,"log_size":7019,"log_dups_size":0,"ondisk_log_size":7019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:59:42.826507+0000","scrub_duration":539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27000832,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":412,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4380,"num_write_kb":67136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":412}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.940749+0000","last_change":"2024-09-13T14:59:12.712394+0000","last_active":"2024-09-13T14:59:50.940749+0000","last_peered":"2024-09-13T14:59:50.940749+0000","last_clean":"2024-09-13T14:59:50.940749+0000","last_became_active":"2024-09-13T14:59:12.711381+0000","last_became_peered":"2024-09-13T14:59:12.711381+0000","last_unstale":"2024-09-13T14:59:50.940749+0000","last_undegraded":"2024-09-13T14:59:50.940749+0000","last_fullsized":"2024-09-13T14:59:50.940749+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:39:28.567717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":333,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919826+0000","last_change":"2024-09-13T14:59:12.712703+0000","last_active":"2024-09-13T14:59:50.919826+0000","last_peered":"2024-09-13T14:59:50.919826+0000","last_clean":"2024-09-13T14:59:50.919826+0000","last_became_active":"2024-09-13T14:59:12.688810+0000","last_became_peered":"2024-09-13T14:59:12.688810+0000","last_unstale":"2024-09-13T14:59:50.919826+0000","last_undegraded":"2024-09-13T14:59:50.919826+0000","last_fullsized":"2024-09-13T14:59:50.919826+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:49:51.655399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.12","version":"335'8994","reported_seq":17290,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.011124+0000","last_change":"2024-09-13T14:59:22.767421+0000","last_active":"2024-09-13T15:00:07.011124+0000","last_peered":"2024-09-13T15:00:07.011124+0000","last_clean":"2024-09-13T15:00:07.011124+0000","last_became_active":"2024-09-13T14:59:22.756767+0000","last_became_peered":"2024-09-13T14:59:22.756767+0000","last_unstale":"2024-09-13T15:00:07.011124+0000","last_undegraded":"2024-09-13T15:00:07.011124+0000","last_fullsized":"2024-09-13T15:00:07.011124+0000","mapping_epoch":331,"log_start":"302'1557","ondisk_log_start":"302'1557","created":307,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"312'4931","last_scrub_stamp":"2024-09-13T14:57:52.187507+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:57:52.187507+0000","objects_scrubbed":2568,"log_size":7437,"log_dups_size":0,"ondisk_log_size":7437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:12:50.874388+0000","scrub_duration":433,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25755648,"num_objects":393,"num_object_clones":0,"num_object_copies":1179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4063,"num_write_kb":60416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2715,"num_bytes_recovered":177930240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":334,"state":"active+clean+remapped","last_fresh":"2024-09-13T14:59:50.941127+0000","last_change":"2024-09-13T14:59:13.726924+0000","last_active":"2024-09-13T14:59:50.941127+0000","last_peered":"2024-09-13T14:59:50.941127+0000","last_clean":"2024-09-13T14:59:50.941127+0000","last_became_active":"2024-09-13T14:59:13.726583+0000","last_became_peered":"2024-09-13T14:59:13.726583+0000","last_unstale":"2024-09-13T14:59:50.941127+0000","last_undegraded":"2024-09-13T14:59:50.941127+0000","last_fullsized":"2024-09-13T14:59:50.941127+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":327,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:01:36.427735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,2],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":318,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.919641+0000","last_change":"2024-09-13T14:59:12.722277+0000","last_active":"2024-09-13T14:59:50.919641+0000","last_peered":"2024-09-13T14:59:50.919641+0000","last_clean":"2024-09-13T14:59:50.919641+0000","last_became_active":"2024-09-13T14:59:12.712060+0000","last_became_peered":"2024-09-13T14:59:12.712060+0000","last_unstale":"2024-09-13T14:59:50.919641+0000","last_undegraded":"2024-09-13T14:59:50.919641+0000","last_fullsized":"2024-09-13T14:59:50.919641+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:42.992912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"12.11","version":"335'9594","reported_seq":18025,"reported_epoch":335,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:00:05.887148+0000","last_change":"2024-09-13T14:59:13.894486+0000","last_active":"2024-09-13T15:00:05.887148+0000","last_peered":"2024-09-13T15:00:05.887148+0000","last_clean":"2024-09-13T14:59:11.679867+0000","last_became_active":"2024-09-13T14:59:13.871230+0000","last_became_peered":"2024-09-13T14:59:13.871230+0000","last_unstale":"2024-09-13T15:00:05.887148+0000","last_undegraded":"2024-09-13T14:59:13.704564+0000","last_fullsized":"2024-09-13T14:59:13.681174+0000","mapping_epoch":326,"log_start":"0'0","ondisk_log_start":"0'0","created":307,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:56:42.793866+0000","objects_scrubbed":0,"log_size":9594,"log_dups_size":0,"ondisk_log_size":9594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27688960,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3803,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7147,"num_write_kb":242208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2782,"num_bytes_recovered":182190080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,3],"avail_no_missing":["5","2","3","6","7"],"object_location_counts":[{"shards":"2,3,5","objects":442},{"shards":"3,5","objects":-1700}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":547,"reported_epoch":336,"state":"active+undersized","last_fresh":"2024-09-13T15:00:07.005504+0000","last_change":"2024-09-13T14:58:00.209414+0000","last_active":"2024-09-13T15:00:07.005504+0000","last_peered":"2024-09-13T15:00:07.005504+0000","last_clean":"2024-09-13T14:57:53.144644+0000","last_became_active":"2024-09-13T14:58:00.209414+0000","last_became_peered":"2024-09-13T14:58:00.209414+0000","last_unstale":"2024-09-13T15:00:07.005504+0000","last_undegraded":"2024-09-13T15:00:07.005504+0000","last_fullsized":"2024-09-13T14:58:00.192721+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:03:41.274290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":519,"reported_epoch":336,"state":"active+clean","last_fresh":"2024-09-13T15:00:07.005756+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:00:07.005756+0000","last_peered":"2024-09-13T15:00:07.005756+0000","last_clean":"2024-09-13T15:00:07.005756+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:00:07.005756+0000","last_undegraded":"2024-09-13T15:00:07.005756+0000","last_fullsized":"2024-09-13T15:00:07.005756+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"12.10","version":"337'9282","reported_seq":15069,"reported_epoch":337,"state":"active+clean+remapped","last_fresh":"2024-09-13T15:00:09.712593+0000","last_change":"2024-09-13T14:59:19.419791+0000","last_active":"2024-09-13T15:00:09.712593+0000","last_peered":"2024-09-13T15:00:09.712593+0000","last_clean":"2024-09-13T15:00:09.712593+0000","last_became_active":"2024-09-13T14:58:41.166787+0000","last_became_peered":"2024-09-13T14:58:41.166787+0000","last_unstale":"2024-09-13T15:00:09.712593+0000","last_undegraded":"2024-09-13T15:00:09.712593+0000","last_fullsized":"2024-09-13T15:00:09.712593+0000","mapping_epoch":320,"log_start":"302'1500","ondisk_log_start":"302'1500","created":307,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":5,"last_scrub":"330'7721","last_scrub_stamp":"2024-09-13T14:59:19.419459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:56:42.793866+0000","last_clean_scrub_stamp":"2024-09-13T14:59:19.419459+0000","objects_scrubbed":1924,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:41:51.203762+0000","scrub_duration":933,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22806528,"num_objects":348,"num_object_clones":0,"num_object_copies":1044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":348,"num_objects_unfound":0,"num_objects_dirty":348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1576,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,2],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":348}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":20,"reported_epoch":334,"state":"active+clean","last_fresh":"2024-09-13T14:59:50.942425+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T14:59:50.942425+0000","last_peered":"2024-09-13T14:59:50.942425+0000","last_clean":"2024-09-13T14:59:50.942425+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T14:59:50.942425+0000","last_undegraded":"2024-09-13T14:59:50.942425+0000","last_fullsized":"2024-09-13T14:59:50.942425+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":187,"reported_epoch":335,"state":"active+undersized+degraded","last_fresh":"2024-09-13T14:59:59.964738+0000","last_change":"2024-09-13T14:58:00.200730+0000","last_active":"2024-09-13T14:59:59.964738+0000","last_peered":"2024-09-13T14:59:59.964738+0000","last_clean":"2024-09-13T14:54:54.202829+0000","last_became_active":"2024-09-13T14:58:00.200730+0000","last_became_peered":"2024-09-13T14:58:00.200730+0000","last_unstale":"2024-09-13T14:59:59.964738+0000","last_undegraded":"2024-09-13T14:58:00.190765+0000","last_fullsized":"2024-09-13T14:58:00.190041+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:35:43.150558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":12,"num_pg":26,"stat_sum":{"num_bytes":809795608,"num_objects":12357,"num_object_clones":0,"num_object_copies":37071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11889,"num_objects_misplaced":1071,"num_objects_unfound":0,"num_objects_dirty":12357,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172147,"num_write_kb":4436449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8583,"num_bytes_recovered":490930176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5242892288,"data_stored":5242880072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":202279,"ondisk_log_size":202279,"up":68,"acting":66,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":87,"acting":89,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":6346,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":16774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13708,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":84,"acting":90,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":20,"acting":21,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":29,"acting":30,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234480,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1883556,"kb_used_data":733468,"kb_used_omap":1,"kb_used_meta":1150078,"kb_avail":91861596,"statfs":{"total":95995035648,"available":94066274304,"internally_reserved":0,"allocated":751071232,"data_stored":749226857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1177680331},"hb_peers":[1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496118,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1818076,"kb_used_data":718036,"kb_used_omap":1,"kb_used_meta":1100030,"kb_avail":91927076,"statfs":{"total":95995035648,"available":94133325824,"internally_reserved":0,"allocated":735268864,"data_stored":733432620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1126431175},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757756,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1341076,"kb_used_data":178700,"kb_used_omap":1,"kb_used_meta":1162366,"kb_avail":92404076,"statfs":{"total":95995035648,"available":94621773824,"internally_reserved":0,"allocated":182988800,"data_stored":181156787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1190263238},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019392,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1087840,"kb_used_data":128088,"kb_used_omap":1,"kb_used_meta":959742,"kb_avail":92657312,"statfs":{"total":95995035648,"available":94881087488,"internally_reserved":0,"allocated":131162112,"data_stored":129321829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":982776262},"hb_peers":[1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650604,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290000,"kb_used_data":478280,"kb_used_omap":4,"kb_used_meta":811643,"kb_avail":92455152,"statfs":{"total":95995035648,"available":94674075648,"internally_reserved":0,"allocated":489758720,"data_stored":487930565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":831122991},"hb_peers":[1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879539,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1350172,"kb_used_data":643220,"kb_used_omap":6,"kb_used_meta":706937,"kb_avail":92394980,"statfs":{"total":95995035648,"available":94612459520,"internally_reserved":0,"allocated":658657280,"data_stored":656817477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":723904057},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,1,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435556,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1269804,"kb_used_data":249188,"kb_used_omap":5,"kb_used_meta":1020602,"kb_avail":92475348,"statfs":{"total":95995035648,"available":94694756352,"internally_reserved":0,"allocated":255168512,"data_stored":253328201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1045096801},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":94,"seq":403726926105,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2675416,"kb_used_data":2196816,"kb_used_omap":5,"kb_used_meta":478586,"kb_avail":91069736,"statfs":{"total":95995035648,"available":93255409664,"internally_reserved":0,"allocated":2249539584,"data_stored":2247846175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5974,"internal_metadata":490072234},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4380,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2269839360,"data_stored":2269839360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":243863552,"data_stored":243859480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":635244544,"data_stored":635240472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":451280896,"data_stored":451280896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":29163520,"data_stored":29163520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":167313408,"data_stored":167313408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":715456512,"data_stored":715456512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":12,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":730730496,"data_stored":730726424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:00:14.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_7 pgp_num 26 2024-09-13T15:00:14.782 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:14.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:14.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:14.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:14.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:14.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:14.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:15.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:15.086 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:15.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:15.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:15.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:15.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:15.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:15.349 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:15.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:15.491 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:15.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:15.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:15.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:15.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:15.823 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:15.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:15.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:15.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:15.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:15.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:16.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.205 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:16.245 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:16.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:16.347 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:16 smithi042 ceph-mon[113926]: pgmap v1212: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 8340/25617 objects degraded (32.557%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:16 smithi042 ceph-mon[108570]: pgmap v1212: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 8340/25617 objects degraded (32.557%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:16.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:16.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:16 smithi159 ceph-mon[102519]: pgmap v1212: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 533 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 896 op/s; 8340/25617 objects degraded (32.557%); 882/25617 objects misplaced (3.443%); 2.8 MiB/s, 100 objects/s recovering 2024-09-13T15:00:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 8340/25617 objects degraded (32.557%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:16.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:16.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:16.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:16.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:16.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:16.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:16.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:16.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:16.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:16.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:17.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.071 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:17.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:17.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:17.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:17.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:17.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:17.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:17.599 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:17.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:17.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:17.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:17.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:17.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:17.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:17.985 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:17.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:17.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:18.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:18.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:18.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:18.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:18.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:18 smithi042 ceph-mon[113926]: pgmap v1213: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 347 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7724/16701 objects degraded (46.249%); 641/16701 objects misplaced (3.838%); 2.8 MiB/s, 123 objects/s recovering 2024-09-13T15:00:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:18 smithi042 ceph-mon[108570]: pgmap v1213: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 347 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7724/16701 objects degraded (46.249%); 641/16701 objects misplaced (3.838%); 2.8 MiB/s, 123 objects/s recovering 2024-09-13T15:00:18.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:18.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:18.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:18 smithi159 ceph-mon[102519]: pgmap v1213: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 347 MiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 7724/16701 objects degraded (46.249%); 641/16701 objects misplaced (3.838%); 2.8 MiB/s, 123 objects/s recovering 2024-09-13T15:00:18.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:18.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:18.620 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:18.656 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:18.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:18.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:18.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:18.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:18.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:18.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:19.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.148 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 116888 objects 2024-09-13T15:00:19.148 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :115.421 2024-09-13T15:00:19.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:19.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:19.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:19.435 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:19.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:19.559 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:19.582 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:19.595 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:19.595 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:19.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:19.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:19.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:19.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:19.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:19] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.51.0" 2024-09-13T15:00:19.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:19.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:19.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:20.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:20.177 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:20.203 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:20.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:20.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:20 smithi042 ceph-mon[108570]: pgmap v1214: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 201 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 6403/9681 objects degraded (66.140%); 329/9681 objects misplaced (3.398%); 2.5 MiB/s, 112 objects/s recovering 2024-09-13T15:00:20.303 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:20.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:20 smithi042 ceph-mon[113926]: pgmap v1214: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 201 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 6403/9681 objects degraded (66.140%); 329/9681 objects misplaced (3.398%); 2.5 MiB/s, 112 objects/s recovering 2024-09-13T15:00:20.303 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:20.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:20.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:20.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:00:20.366 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:20.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.449 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:20 smithi159 ceph-mon[102519]: pgmap v1214: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 201 MiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 6403/9681 objects degraded (66.140%); 329/9681 objects misplaced (3.398%); 2.5 MiB/s, 112 objects/s recovering 2024-09-13T15:00:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:00:20.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:20.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:20.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:20.701 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:20.803 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:20.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:20.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:20.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:20.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:20.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:21.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:21.074 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:21.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:21.080 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:21 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 6403/9681 objects degraded (66.140%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:21.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:21.198 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:21.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:21.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:21.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:21.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:21 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 6403/9681 objects degraded (66.140%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:21.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:21.438 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:21 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 6403/9681 objects degraded (66.140%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:21.438 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:21.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:21.551 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.560 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:21.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:21.680 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:21.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:21.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:21.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:21.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:21.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:21.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:21.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:21.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:22.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.088 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:00:22.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:22.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:22.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:22.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:22.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[108570]: pgmap v1215: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 135 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 6130/6537 objects degraded (93.774%); 146/6537 objects misplaced (2.233%); 2.2 MiB/s, 98 objects/s recovering 2024-09-13T15:00:22.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4287140324' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[113926]: pgmap v1215: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 135 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 6130/6537 objects degraded (93.774%); 146/6537 objects misplaced (2.233%); 2.2 MiB/s, 98 objects/s recovering 2024-09-13T15:00:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4287140324' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:22 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.392 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:22 smithi159 ceph-mon[102519]: pgmap v1215: 109 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized, 11 active+undersized+degraded, 68 active+clean; 135 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 935 op/s; 6130/6537 objects degraded (93.774%); 146/6537 objects misplaced (2.233%); 2.2 MiB/s, 98 objects/s recovering 2024-09-13T15:00:22.392 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4287140324' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.392 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:22 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:22.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:22.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:22.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:22.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:22.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:22.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:22.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:23.047 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:23.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:23.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:23.296 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:23.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:23.347 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:23 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:23.347 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:23 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:23.347 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:23 smithi159 ceph-mon[102519]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T15:00:23.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:23.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:23.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[108570]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T15:00:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:23.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:23 smithi042 ceph-mon[113926]: osdmap e338: 8 total, 7 up, 8 in 2024-09-13T15:00:23.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:23.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:23.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:23.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:23.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:23.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:23.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:23.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:23.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:23.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:23.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:23.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.042 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:24.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:24.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:24.166 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:24.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:24.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:24.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:24.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:24 smithi042 ceph-mon[108570]: pgmap v1217: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:24.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:24 smithi042 ceph-mon[113926]: pgmap v1217: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:24.398 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:24 smithi159 ceph-mon[102519]: pgmap v1217: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:24.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:24.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:24.543 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:24.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:24.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:24.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:24.792 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:24.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:24.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:24.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:24.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:24.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:25.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:25.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:25.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:25.092 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 12 pgp_num to 26 2024-09-13T15:00:25.165 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:25.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:25.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:25.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:25.312 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:25.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:25.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T15:00:25.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T15:00:25.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:25.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"} : dispatch 2024-09-13T15:00:25.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:25.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:25.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:25.676 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:25.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:25.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:25.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:25.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:25.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:25.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:25.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:25.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:25.972 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:26.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.199 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:26.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: pgmap v1218: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5335/300 objects degraded (1778.333%), 9 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: pgmap v1218: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5335/300 objects degraded (1778.333%), 9 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.409 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_7 2024-09-13T15:00:26.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_7 unique_pool_7 --yes-i-really-really-mean-it 2024-09-13T15:00:26.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:26.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:26.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: pgmap v1218: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 7 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 5335/300 objects degraded (1778.333%); 3/300 objects misplaced (1.000%); 486 KiB/s, 94 objects/s recovering 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/154290112' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_7", "var": "pgp_num", "val": "26"}]': finished 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: osdmap e339: 8 total, 7 up, 8 in 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5335/300 objects degraded (1778.333%), 9 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:26.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.598 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:26.621 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:00:26.621 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T15:00:26.621 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:26.642 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:00:26.643 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:26.660 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:00:26.660 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:00:26.676 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:00:26.677 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:26.693 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:00:26.693 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:26.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:26.710 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:00:26.710 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:26.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:26.728 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:00:26.728 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:26.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:26.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:26.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:26.972 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:26.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:27.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:27.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:27.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:27.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:27.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:27.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:27 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4181716363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:27.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:27 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4181716363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:27.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:27.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.347 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:27.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:27.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:27.471 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:27.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:27 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4181716363' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:27.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:00:27.573 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:27.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:27.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:27.697 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:27.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:27.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:27.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:27.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:27.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:27.946 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:27.968 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:27.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:28.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:28.094 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:28.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:28.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.220 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:28.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:28.321 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:28.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:28.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:28.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:28 smithi042 ceph-mon[113926]: pgmap v1220: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+undersized, 4 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:28.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:28 smithi042 ceph-mon[108570]: pgmap v1220: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+undersized, 4 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:28.376 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:28.447 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:28.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:28.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:28 smithi159 ceph-mon[102519]: pgmap v1220: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 23 active+undersized, 4 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:28.523 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:28.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.594 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:28.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:28.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:28.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:28.839 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:28.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:28.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:28.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:28.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:29.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:29.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:29.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:29.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:29.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:29.448 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:29.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:29.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:29.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:29.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:29.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:29.843 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:29.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:29.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:29.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:29.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:30.027 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:29] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-09-13T15:00:30.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.092 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.216 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:30.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:30.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:30.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:30.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:30 smithi042 ceph-mon[113926]: pgmap v1221: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 262 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:30.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:30 smithi042 ceph-mon[113926]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T15:00:30.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:30 smithi042 ceph-mon[108570]: pgmap v1221: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 262 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:30 smithi042 ceph-mon[108570]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T15:00:30.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:00:30.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:30.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:30.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:30 smithi159 ceph-mon[102519]: pgmap v1221: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 262 op/s; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 202 objects/s recovering 2024-09-13T15:00:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:30 smithi159 ceph-mon[102519]: osdmap e340: 8 total, 7 up, 8 in 2024-09-13T15:00:30.569 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:30.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:30.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:30.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:30.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:30.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:30.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:30.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:30.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:30.924 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:30.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:30.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:30.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:30.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:31.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:31.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.216 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:31.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:31.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:31.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:31.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:31 smithi042 ceph-mon[108570]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T15:00:31.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 5087/300 objects degraded (1695.667%), 5 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:31 smithi042 ceph-mon[113926]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T15:00:31.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 5087/300 objects degraded (1695.667%), 5 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:31.409 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [0] live_osds: [1, 2, 3, 4, 5, 6, 7] 2024-09-13T15:00:31.409 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:00:31.410 INFO:tasks.thrashosds.thrasher:Reviving osd 0 2024-09-13T15:00:31.410 INFO:tasks.cephadm.osd.0:Restarting osd.0 (starting--it wasn't running)... 2024-09-13T15:00:31.410 DEBUG:teuthology.orchestra.run.smithi042:osd.0> sudo journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T15:00:31.412 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl start ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0 2024-09-13T15:00:31.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:31.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:31 smithi159 ceph-mon[102519]: osdmap e341: 8 total, 7 up, 8 in 2024-09-13T15:00:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 5087/300 objects degraded (1695.667%), 5 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:31.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:31.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:31.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:31.683 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:00:31.683 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:00:31.683 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:00:31.683 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:00:31.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:31.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:31.815 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:31.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:31.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:31.848 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:31 smithi042 systemd[1]: Starting Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:00:31.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:31.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:31.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:31.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:32.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:32.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:32.187 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:32.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:32.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:32.329 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:32 smithi042 ceph-mon[108570]: pgmap v1224: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 175 objects/s recovering 2024-09-13T15:00:32.329 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:32 smithi042 ceph-mon[113926]: pgmap v1224: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 175 objects/s recovering 2024-09-13T15:00:32.335 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:32.359 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:00:32.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.465 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:32 smithi159 ceph-mon[102519]: pgmap v1224: 109 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 68 active+clean; 5.3 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5087/300 objects degraded (1695.667%); 3/300 objects misplaced (1.000%); 0 B/s, 175 objects/s recovering 2024-09-13T15:00:32.566 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:32.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:32.601 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:32 smithi042 podman[281965]: 2024-09-13 15:00:32.362990265 +0000 UTC m=+0.019335566 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:00:32.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:32.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:32.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:32.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:32.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:32.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:32.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.090 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:33.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:33.201 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:32 smithi042 podman[281965]: 2024-09-13 15:00:32.906086342 +0000 UTC m=+0.562431624 container create 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:00:33.201 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:33 smithi042 podman[281965]: 2024-09-13 15:00:33.200550798 +0000 UTC m=+0.856896080 container init 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T15:00:33.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:33.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:33.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:33.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:33.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.465 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:33.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:33.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:33 smithi042 podman[281965]: 2024-09-13 15:00:33.205591586 +0000 UTC m=+0.861936877 container start 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T15:00:33.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:33 smithi042 podman[281965]: 2024-09-13 15:00:33.35049077 +0000 UTC m=+1.006836056 container attach 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:00:33.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:33.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:33.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:33.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:33.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:33.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:33.962 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:33.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:34.063 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:34.186 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:34.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:34.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:34 smithi042 ceph-mon[113926]: pgmap v1225: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 408 objects/s recovering 2024-09-13T15:00:34.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:34 smithi042 ceph-mon[108570]: pgmap v1225: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 408 objects/s recovering 2024-09-13T15:00:34.265 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:34.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.329 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:34.431 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:34.452 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:34 smithi159 ceph-mon[102519]: pgmap v1225: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 408 objects/s recovering 2024-09-13T15:00:34.554 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:34.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:34.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:34.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:34.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:34.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:34.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:34.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:34.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:35.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.078 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.126 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_7' removed 2024-09-13T15:00:35.144 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:00:35.144 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:00:35.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:35.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:35.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:35.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:35.409 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:35.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:35.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:35.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:00:35.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:35.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:35.694 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:35.717 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:35.786 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.786 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.786 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.786 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.787 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T15:00:35.819 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.841 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:35.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:35.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:35.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:35.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:36.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:36.110 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T15:00:36.110 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T15:00:36.110 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T15:00:36.110 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:35 smithi042 bash[281965]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-13T15:00:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:36.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:36.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:36.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:36.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:36.343 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:00:36.344 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:00:36.344 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_8 2024-09-13T15:00:36.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_8 16 2024-09-13T15:00:36.358 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:36.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:36.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:36 smithi159 ceph-mon[102519]: pgmap v1226: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 251 objects/s recovering 2024-09-13T15:00:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:36 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:00:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:36 smithi159 ceph-mon[102519]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T15:00:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:36 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2048/300 objects degraded (682.667%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:36.535 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[113926]: pgmap v1226: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 251 objects/s recovering 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[113926]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2048/300 objects degraded (682.667%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[108570]: pgmap v1226: 109 pgs: 1 active+undersized+remapped+backfill_wait, 11 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 24 active+undersized, 3 active+undersized+degraded, 69 active+clean; 5.3 MiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 2048/300 objects degraded (682.667%); 3/300 objects misplaced (1.000%); 0 B/s, 251 objects/s recovering 2024-09-13T15:00:36.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/837487305' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_7", "pool2": "unique_pool_7", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:00:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[108570]: osdmap e342: 8 total, 7 up, 8 in 2024-09-13T15:00:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2048/300 objects degraded (682.667%), 4 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:36.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 bash[281965]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 bash[281965]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 bash[281965]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T15:00:36.600 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 bash[281965]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T15:00:36.601 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate[282094]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T15:00:36.601 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 bash[281965]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-13T15:00:36.601 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:36 smithi042 podman[281965]: 2024-09-13 15:00:36.31993915 +0000 UTC m=+3.976284447 container died 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T15:00:36.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:36.660 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:36.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:36.762 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:36.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:36.886 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:36.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:36.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:37.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:37.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:37.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:37.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:37.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.385 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:37.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:37.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:37.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:37.534 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:37.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:37.636 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:37.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:37.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.789 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:37.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:37.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:37.915 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:37.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:38.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:38.041 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:38.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:38.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:38.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:38.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:38.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:38.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:38 smithi042 ceph-mon[108570]: pgmap v1228: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:38.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:38 smithi042 ceph-mon[113926]: pgmap v1228: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:38.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:38.420 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:38 smithi159 ceph-mon[102519]: pgmap v1228: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:38.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:38.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:38.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:38.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:38.778 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:38.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:38.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:38.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:38.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:38.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:39.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:39.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:39.154 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.177 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:39.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:39.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:39.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:39.409 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:39.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:39.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:39.562 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:39.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:39.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:39.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:39.789 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:39.813 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:39.915 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:39.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:39.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:40.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:40.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:39] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.51.0" 2024-09-13T15:00:40.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.192 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:40.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:40.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:40.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:40.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:40 smithi159 ceph-mon[102519]: pgmap v1229: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:40.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:40.571 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:40 smithi042 ceph-mon[113926]: pgmap v1229: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:40 smithi042 ceph-mon[108570]: pgmap v1229: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:40.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:40.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:40.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:40.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:40.955 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:40.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:41.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:41.084 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:41.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:41.187 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:41.213 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:41.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:41.315 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:41.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:41.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:41.573 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:41.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-09-13T15:00:41.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:41.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:41.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:41.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:41.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:41.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:41.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:41.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:41.982 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:41.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:42.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:42.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:42.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:42 smithi042 ceph-mon[113926]: pgmap v1230: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:42.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:42 smithi042 ceph-mon[108570]: pgmap v1230: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:42.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:42.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.339 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:42.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:42.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:42.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:42 smithi159 ceph-mon[102519]: pgmap v1230: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:42.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:42.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:42.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:42.715 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:42.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:42.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:42.865 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:42.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:42.893 INFO:teuthology.orchestra.run.smithi042.stderr:Error ENXIO: problem getting command descriptions from osd.0 2024-09-13T15:00:42.897 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:00:42.897 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:00:42.914 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:00:42.915 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:00:42.930 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:00:42.931 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:00:42.948 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:00:42.948 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:00:42.965 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:00:42.965 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:00:42.967 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:42.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:00:42.981 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:00:42.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:42.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:42.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.004 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:00:43.004 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:00:43.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.142 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:00:43.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:43.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:43.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3014562787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3014562787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:43.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:43.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:43.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:43.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:00:43.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3014562787' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:00:43.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:43.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:43.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:43.628 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:43.730 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:43.753 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:43.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:43.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:43.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:43.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:43.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:44.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:44.135 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:44.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:44.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:44.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:44.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:44.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:43 smithi042 podman[282671]: 2024-09-13 15:00:43.983231814 +0000 UTC m=+7.671649208 container remove 6e9bb1d4a58aeb05c2f32b0a3a73f86362a7accb423811c7be49b1ece39b8c23 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T15:00:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[108570]: pgmap v1231: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[108570]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T15:00:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[113926]: pgmap v1231: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:44.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:44 smithi042 ceph-mon[113926]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T15:00:44.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:44.387 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:44.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:44.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:44 smithi159 ceph-mon[102519]: pgmap v1231: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:44 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:00:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1630037368' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:00:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:44 smithi159 ceph-mon[102519]: osdmap e343: 8 total, 7 up, 8 in 2024-09-13T15:00:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:44.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:44.615 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:44.638 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:44.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:44.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:44.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:44.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:44.890 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:44.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:44.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:44.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:45.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:45.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:45.141 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:45.242 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:45.245 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:45 smithi042 podman[283601]: 2024-09-13 15:00:44.997845343 +0000 UTC m=+0.446432594 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:00:45.245 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:45 smithi042 podman[283601]: 2024-09-13 15:00:45.1072251 +0000 UTC m=+0.555812336 container create fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:00:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.266 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:45.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:45.367 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:45.390 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:45.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:45.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.515 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:45.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:45.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:45.639 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:45.740 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:45.753 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:45.766 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:45.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:45.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:45.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:45.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:45.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:46.021 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:46.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.149 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:46.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:00:46.250 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:46.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:46.367 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:46 smithi042 ceph-mon[113926]: pgmap v1233: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:46.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:46 smithi042 ceph-mon[108570]: pgmap v1233: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:46.368 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:46 smithi042 podman[283601]: 2024-09-13 15:00:46.107316168 +0000 UTC m=+1.555903406 container init fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS) 2024-09-13T15:00:46.368 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:46 smithi042 podman[283601]: 2024-09-13 15:00:46.113208205 +0000 UTC m=+1.561795441 container start fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T15:00:46.368 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:46 smithi042 ceph-osd[283819]: -- 172.21.15.42:0/3929339456 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557c22c0da40 con 0x557c239da400 2024-09-13T15:00:46.368 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:46 smithi042 bash[283601]: fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a 2024-09-13T15:00:46.369 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:46 smithi042 systemd[1]: Started Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:00:46.382 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.407 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.475 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:00:46.475 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:00:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:46 smithi159 ceph-mon[102519]: pgmap v1233: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:46.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.531 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:46.654 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:46.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:46.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:46.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:46.900 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:46.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:46.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:46.995 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:00:47.002 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:47.023 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:47.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:47.272 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:47.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:47.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:47.497 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:47.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:47.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:47.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:47.644 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:47.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:47.746 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:47.769 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:47.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:47.849 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:47 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:00:47.514+0000 7f1ce430e640 -1 Falling back to public interface 2024-09-13T15:00:47.871 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:47.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:47.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:47.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:47.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:48.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:48.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:48.119 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:48.142 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:48.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:48.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:48 smithi042 ceph-mon[108570]: pgmap v1234: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:48 smithi042 ceph-mon[113926]: pgmap v1234: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:48.369 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:48.391 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:48.493 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:48 smithi159 ceph-mon[102519]: pgmap v1234: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:48.514 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:48.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:00:48.617 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:48.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:48.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:48.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:48.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:48.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:48.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:48.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:48.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:49.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-13T15:00:49.031 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-13T15:00:49.039 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:49.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:49.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:49.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:49.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:49.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:49.289 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:49.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:49.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:49.414 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:49.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:49.516 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:49.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:49.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:49.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:49.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:49.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:49.794 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:49.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:49.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:49] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.51.0" 2024-09-13T15:00:49.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:49.919 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:49.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:50.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:50.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:50.148 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:50.150 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_8' created 2024-09-13T15:00:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:50.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:50.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:50.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.306 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:50.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:50.360 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:50.360 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:50.408 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:50.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:50.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:50.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[108570]: pgmap v1235: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:50.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1414177127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[113926]: pgmap v1235: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:50.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1414177127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.481 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:50 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:50 smithi159 ceph-mon[102519]: pgmap v1235: 83 pgs: 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1414177127' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:50 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16} : dispatch 2024-09-13T15:00:50.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:50.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:50.658 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:50.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:50.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:50.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:50.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:50.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_8 rados --yes-i-really-mean-it || true 2024-09-13T15:00:50.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:50.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:50.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:50.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:51.033 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:00:51.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:51.099 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:51.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:51.235 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:51.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:51.337 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:51.363 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:51.451 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-13T15:00:51.451 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[113926]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T15:00:51.451 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.451 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[113926]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T15:00:51.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-13T15:00:51.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[108570]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T15:00:51.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.452 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:51 smithi042 ceph-mon[108570]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T15:00:51.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:51.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:51 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_8", "pg_num": 16}]': finished 2024-09-13T15:00:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:51 smithi159 ceph-mon[102519]: osdmap e344: 8 total, 7 up, 8 in 2024-09-13T15:00:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:00:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:51 smithi159 ceph-mon[102519]: osdmap e345: 8 total, 7 up, 8 in 2024-09-13T15:00:51.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:51.614 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:51.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:51.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:51.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:51.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:51.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:51.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:51.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:51.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:52.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:52.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:52.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:52.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:52.241 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:52.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:52.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:52.369 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:52.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:52.370 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:52.471 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:52.491 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[108570]: pgmap v1237: 99 pgs: 16 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:52.491 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:00:52.492 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[108570]: osdmap e346: 8 total, 7 up, 8 in 2024-09-13T15:00:52.492 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[113926]: pgmap v1237: 99 pgs: 16 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:52.492 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:00:52.492 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:52 smithi042 ceph-mon[113926]: osdmap e346: 8 total, 7 up, 8 in 2024-09-13T15:00:52.494 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:52 smithi159 ceph-mon[102519]: pgmap v1237: 99 pgs: 16 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 54 active+clean; 582 KiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:52 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:00:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:52 smithi159 ceph-mon[102519]: osdmap e346: 8 total, 7 up, 8 in 2024-09-13T15:00:52.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:52.622 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:52.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:52.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:52.841 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:52.847 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:00:52.848 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:52.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:52.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:52.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.002 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:53.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:53.234 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:53.258 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:53.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:53.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:53.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:53.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:53.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.515 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:53.515 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:53.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:53.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:53.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:53.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:53.769 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:53.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:53.870 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:53.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:53.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:53.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:54.025 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:54.127 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.152 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.254 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:54.269 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:54 smithi042 ceph-mon[108570]: pgmap v1240: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:54.270 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:54 smithi042 ceph-mon[113926]: pgmap v1240: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:54.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:54.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:54.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:54 smithi159 ceph-mon[102519]: pgmap v1240: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:54.508 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:54.534 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:54.636 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:54.789 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:54.891 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:54.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:54.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:54.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.017 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:55.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:55.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:55.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:55.272 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:55.294 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:55.396 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:55.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:55.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:55.598 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:55 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:00:55.247+0000 7f1ce430e640 -1 osd.0 314 log_to_monitors true 2024-09-13T15:00:55.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:55.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:55.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:55.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:55.906 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:55.931 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:55.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:55.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.183 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:56.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:56 smithi042 ceph-mon[113926]: pgmap v1241: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:56.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:56 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T15:00:56.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:56 smithi042 ceph-mon[108570]: pgmap v1241: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:56.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:56 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T15:00:56.288 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:00:56 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:00:56.196+0000 7f1cdc0b8640 -1 osd.0 314 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T15:00:56.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:56.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.436 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:56.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:56 smithi159 ceph-mon[102519]: pgmap v1241: 99 pgs: 12 creating+peering, 2 unknown, 10 active+clean+remapped, 16 active+undersized, 3 active+undersized+degraded, 56 active+clean; 582 KiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:56 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T15:00:56.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:56.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:56.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:56.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:56.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:56.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:56.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:56.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:56.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:56.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:56.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:57.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:57.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:57.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.176 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:00:57.196 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:57.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.305 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T15:00:57.305 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[113926]: osdmap e347: 8 total, 7 up, 8 in 2024-09-13T15:00:57.305 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[113926]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T15:00:57.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:57.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T15:00:57.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[108570]: osdmap e347: 8 total, 7 up, 8 in 2024-09-13T15:00:57.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[108570]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T15:00:57.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:57 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:57.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:57.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:57.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:57.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:57.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:57 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T15:00:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:57 smithi159 ceph-mon[102519]: osdmap e347: 8 total, 7 up, 8 in 2024-09-13T15:00:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:57 smithi159 ceph-mon[102519]: from='osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T15:00:57.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:57 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:00:57.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:57.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:57.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:57.676 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:57.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:57.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:57.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:57.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:57.927 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:57.950 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:57.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:57.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:58.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:58.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:58.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:58.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:58.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:58.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:58.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:58.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: pgmap v1243: 99 pgs: 10 active+clean+remapped, 19 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115] boot 2024-09-13T15:00:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T15:00:58.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T15:00:58.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:58 smithi159 ceph-mon[102519]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T15:00:58.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:58.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: pgmap v1243: 99 pgs: 10 active+clean+remapped, 19 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115] boot 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[108570]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: pgmap v1243: 99 pgs: 10 active+clean+remapped, 19 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2945776369' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: osd.0 [v2:172.21.15.42:6802/2799202115,v1:172.21.15.42:6803/2799202115] boot 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: osdmap e348: 8 total, 8 up, 8 in 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:58.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:58 smithi042 ceph-mon[113926]: osdmap e349: 8 total, 8 up, 8 in 2024-09-13T15:00:58.674 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:58.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:58.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:00:58.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:00:58.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:00:58.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:58.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:58.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:00:59.044 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:00:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:59.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:00:59.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.196 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:59.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:59.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:00:59.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:00:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:00:59 smithi159 ceph-mon[102519]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T15:00:59.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:00:59.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:00:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:00:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[108570]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:00:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-mon[113926]: osdmap e350: 8 total, 8 up, 8 in 2024-09-13T15:00:59.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:59.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:59.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:00:59.819 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:00:59.921 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:00:59.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:00:59.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:00:59.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:00:59.979 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:00:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:00:59] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.51.0" 2024-09-13T15:01:00.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:00.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:00.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:00.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:00 smithi042 ceph-mon[113926]: pgmap v1246: 99 pgs: 1 peering, 10 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:00.241 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:00.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:00 smithi042 ceph-mon[108570]: pgmap v1246: 99 pgs: 1 peering, 10 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:00.241 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:00.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:00.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:00.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.444 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:00 smithi159 ceph-mon[102519]: pgmap v1246: 99 pgs: 1 peering, 10 active+clean+remapped, 18 active+undersized, 3 active+undersized+degraded, 67 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2024-09-13T15:01:00.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:00.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:00.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:00.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:00.745 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:01:00.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:00.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:00.854 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:01:00.854 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:01:00.854 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:01:00.854 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:01:00.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:00.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:00.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:00.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:01.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:01.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:01.152 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:01.152 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:01.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.190 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:01.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:01.199 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_8' 2024-09-13T15:01:01.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:01.300 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2648054062' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.300 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:01 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.301 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2648054062' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.301 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:01 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:01.416 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:01.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:01.461 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2648054062' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.461 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:01 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:01:01.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:01.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:01.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:01.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:01.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:01.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:01.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:01.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:01.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:01.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:01.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:01:02.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:02.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:02.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:02.303 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:02.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:02.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[108570]: pgmap v1248: 99 pgs: 1 peering, 10 active+clean+remapped, 15 active+undersized, 3 active+undersized+degraded, 70 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:01:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[108570]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T15:01:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[113926]: pgmap v1248: 99 pgs: 1 peering, 10 active+clean+remapped, 15 active+undersized, 3 active+undersized+degraded, 70 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:01:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:02 smithi042 ceph-mon[113926]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T15:01:02.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:02.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.454 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:02.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:02 smithi159 ceph-mon[102519]: pgmap v1248: 99 pgs: 1 peering, 10 active+clean+remapped, 15 active+undersized, 3 active+undersized+degraded, 70 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 7/78 objects degraded (8.974%); 3/78 objects misplaced (3.846%) 2024-09-13T15:01:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_8", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:01:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:02 smithi159 ceph-mon[102519]: osdmap e351: 8 total, 8 up, 8 in 2024-09-13T15:01:02.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:02.577 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:02.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:02.679 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:02.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:02.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:02.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:02.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:02.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:02.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:02.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:03.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:03.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:03.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:03.175 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:01:03.175 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:01:03.175 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:03.175 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:03.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:03.190 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:01:03.191 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:03.209 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.209 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:01:03.209 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:03.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:03.225 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:01:03.225 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:03.242 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:01:03.242 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:03.259 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:01:03.259 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:03.275 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:01:03.275 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:03.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:03.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:03.437 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:03.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:03.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/58984002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:03.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:03.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/58984002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/58984002' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:03.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:03.717 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:03.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:03.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:03.923 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_8 bench 90 write' 2024-09-13T15:01:03.926 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:01:03.926 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:01:03.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:03.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:03.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:03.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:03.978 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T15:01:03.978 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:01:03.978 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_286876 2024-09-13T15:01:03.979 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:01:03.979 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:01:04.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:04.111 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:04.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:04.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:04.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:04.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:04.349 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:04.384 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:04.397 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:04.397 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:04.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:04 smithi159 ceph-mon[102519]: pgmap v1250: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:04 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 15 pgs undersized) 2024-09-13T15:01:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:04 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:01:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:04 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:01:04.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:04.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:04.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[108570]: pgmap v1250: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 15 pgs undersized) 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[113926]: pgmap v1250: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/78 objects degraded (8.974%), 3 pgs degraded, 15 pgs undersized) 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:01:04.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:04 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:01:04.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:04.660 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:04.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:04.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.785 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:04.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:04.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:04.923 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:04.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:04.979 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3751 3735 233.424 233.438 0.00336785 0.00427281 2024-09-13T15:01:05.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:05.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.063 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:05.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:05.165 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:05.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:05.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:05.344 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:05.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:05.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:05.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:05.622 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:05.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:05.724 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:05.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:05.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:05.862 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:05.897 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:05.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:05.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:05.979 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 7153 7137 223.015 212.625 0.0028299 0.00447527 2024-09-13T15:01:05.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:06.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:06.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:06.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.176 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:06.278 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:06.315 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:06.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:06.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:06.417 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:06.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:06.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:06 smithi159 ceph-mon[102519]: pgmap v1251: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[108570]: pgmap v1251: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[113926]: pgmap v1251: 99 pgs: 99 active+clean; 582 KiB data, 9.1 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:06.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:06.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:06.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:06.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:06.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:06.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:06.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:06.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:06.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:06.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:06.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:06.979 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 10547 10531 219.379 212.125 0.00375218 0.00455091 2024-09-13T15:01:07.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:07.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:07.119 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:07.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:07.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:07.295 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:07.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:07.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:07.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:07.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:07.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:07.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:07.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:07.676 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:07.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:07.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:07.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:07.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:07.980 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 13534 13518 211.116 186.688 0.00583358 0.00472875 2024-09-13T15:01:07.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:07.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:08.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.132 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:08.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.274 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:08.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.394 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:08.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:08.474 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:08 smithi042 ceph-mon[108570]: pgmap v1252: 99 pgs: 99 active+clean; 356 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:01:08.474 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:08 smithi042 ceph-mon[113926]: pgmap v1252: 99 pgs: 99 active+clean; 356 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:01:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:08 smithi159 ceph-mon[102519]: pgmap v1252: 99 pgs: 99 active+clean; 356 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s 2024-09-13T15:01:08.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:08.549 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:08.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:08.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:08.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:08.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:08.820 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:08.842 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:01:08.843 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:01:08.843 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-13T15:01:08.843 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:01:08.843 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:01:08.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-13T15:01:08.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:08.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:08.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:08.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:08.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 16401 16385 204.729 179.188 0.00567272 0.00487798 2024-09-13T15:01:09.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:09.107 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:09.198 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:01:09.198 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:01:09.198 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:01:09.198 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:01:09.199 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:09.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:09.248 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:09.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:09.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:09.487 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:09.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:09.623 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:09.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:09.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:09.796 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:09.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:09.934 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:09.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:09.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 19085 19069 198.565 167.75 0.00502221 0.00503057 2024-09-13T15:01:10.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:10.072 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:09] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-13T15:01:10.162 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:01:10.174 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:10.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:10.374 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:10 smithi042 ceph-mon[108570]: pgmap v1253: 99 pgs: 99 active+clean; 450 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2024-09-13T15:01:10.374 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:10 smithi042 ceph-mon[113926]: pgmap v1253: 99 pgs: 99 active+clean; 450 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2024-09-13T15:01:10.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:10.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:10.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:10 smithi159 ceph-mon[102519]: pgmap v1253: 99 pgs: 99 active+clean; 450 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s 2024-09-13T15:01:10.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:10.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:10.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:10.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:10.791 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:10.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:10.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:10.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:10.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:10.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 21245 21229 189.484 135 0.00408334 0.00527116 2024-09-13T15:01:11.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:11.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:11.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:11.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:11.231 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:01:11.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:11.324 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:11.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:11.447 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:11.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:11 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:11.549 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:11.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:11.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:11 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:11.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:11 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:01:11.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:11.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:11.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:11.828 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:11.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:11.966 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:11.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 23229 23213 181.292 124 0.00542197 0.00551 2024-09-13T15:01:11.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:11.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:11.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:12.099 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:12.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:12.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:12.236 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:12.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:12.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:12.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:12.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:12.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:12 smithi159 ceph-mon[102519]: pgmap v1254: 99 pgs: 99 active+clean; 573 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-13T15:01:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:12 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:12 smithi159 ceph-mon[102519]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T15:01:12.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[108570]: pgmap v1254: 99 pgs: 99 active+clean; 573 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-13T15:01:12.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[108570]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[113926]: pgmap v1254: 99 pgs: 99 active+clean; 573 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4038306552' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:01:12.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:12 smithi042 ceph-mon[113926]: osdmap e352: 8 total, 8 up, 8 in 2024-09-13T15:01:12.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:12.648 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:12.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:12.750 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:12.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:12.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:12.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:12.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:12.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:12.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 25450 25434 176.572 138.812 0.00200389 0.00563971 2024-09-13T15:01:13.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:13.068 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:13.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:13.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:13.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:13.308 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:13.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:13.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:13.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:13.551 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:01:13.551 INFO:teuthology.orchestra.run.smithi159.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-13T15:01:13.551 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:01:13.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:13.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:13.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:13.713 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:13.747 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:13.849 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:13.883 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:13.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:13.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:13.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 27691 27675 172.921 140.062 0.0112105 0.00577755 2024-09-13T15:01:13.985 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.023 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:14.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:14.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:14.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:14.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:14.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:14.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:14.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:14.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:14 smithi159 ceph-mon[102519]: pgmap v1256: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:14.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:14.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:14.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:14.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:14 smithi042 ceph-mon[108570]: pgmap v1256: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:14 smithi042 ceph-mon[113926]: pgmap v1256: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:14.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:14.705 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:14.807 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:14.843 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:14.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:14.979 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:14.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:14.981 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 29830 29814 169.354 133.688 0.00499291 0.0058994 2024-09-13T15:01:15.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.118 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:15.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:15.255 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:15.357 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:15.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.499 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:15.534 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:15.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:15.636 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.673 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:15.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.775 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:15.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:15.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:15.917 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:15.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:15.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:15.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:15.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 32077 32061 166.943 140.438 0.00587128 0.00598491 2024-09-13T15:01:16.000 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:01:16.001 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:16.037 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:01:16.038 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:16.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:16.075 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:01:16.076 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:16.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:16.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:16.111 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:01:16.111 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:16.145 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:01:16.145 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:16.177 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:01:16.178 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:16.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:16.205 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:01:16.206 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:16.235 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:16.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:16.237 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:01:16.237 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:16.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:16 smithi042 ceph-mon[113926]: pgmap v1257: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:16 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1300902980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:16 smithi042 ceph-mon[108570]: pgmap v1257: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:16 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1300902980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:16.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:16.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:16.479 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:16 smithi159 ceph-mon[102519]: pgmap v1257: 99 pgs: 99 active+clean; 1.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-13T15:01:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:16 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1300902980' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:16.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:16.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:16.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:16.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:16.794 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:16.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:16.818 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:16.896 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:16.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:16.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:16.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 34115 34099 163.899 127.375 0.00795427 0.00609503 2024-09-13T15:01:17.032 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:17.070 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:17.172 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:17.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:17.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:17.344 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:17.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:17.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:17.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:17.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:17.621 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:17.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:17.723 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:17.759 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:17.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:17.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:17.902 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:17.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:17.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 35992 35976 160.572 117.312 0.00733586 0.00622245 2024-09-13T15:01:18.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:18.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:18.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:18.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:18.307 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:18.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:18.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:18.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:18 smithi159 ceph-mon[102519]: pgmap v1258: 99 pgs: 99 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-13T15:01:18.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:01:18.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:18 smithi042 ceph-mon[108570]: pgmap v1258: 99 pgs: 99 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-13T15:01:18.597 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:18 smithi042 ceph-mon[113926]: pgmap v1258: 99 pgs: 99 active+clean; 1.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 156 MiB/s wr, 2.50k op/s 2024-09-13T15:01:18.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:18.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:18.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:18.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:18.863 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:18.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:18.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:18.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 37939 37923 157.978 121.688 0.00754665 0.00632422 2024-09-13T15:01:19.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:19.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:19.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:19.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:19.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:19.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:19.269 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:19.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:19.316 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:01:19.316 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:01:19.316 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-13T15:01:19.316 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.000000 2024-09-13T15:01:19.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 0 2024-09-13T15:01:19.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:19.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.434 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:19.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:19.535 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:19.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:19.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:19.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:19.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:19.801 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:19.802 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:19.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:19.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:19.871 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:19.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:19.972 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:19.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 39830 39814 155.49 118.188 0.006039 0.00642599 2024-09-13T15:01:20.005 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:20.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:20.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:19] "GET /metrics HTTP/1.1" 200 36462 "" "Prometheus/2.51.0" 2024-09-13T15:01:20.107 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:20.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:20.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:20.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:20.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:20.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:20.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:20.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:20.570 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:20.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:20.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:20 smithi042 ceph-mon[108570]: pgmap v1259: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 155 MiB/s wr, 2.49k op/s 2024-09-13T15:01:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:20 smithi042 ceph-mon[113926]: pgmap v1259: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 155 MiB/s wr, 2.49k op/s 2024-09-13T15:01:20.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:20.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:20.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:20 smithi159 ceph-mon[102519]: pgmap v1259: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 155 MiB/s wr, 2.49k op/s 2024-09-13T15:01:20.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:20.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:20.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:20.964 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:20.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:20.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 41496 41480 152.468 104.125 0.00234853 0.00649468 2024-09-13T15:01:21.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:21.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:21.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:21.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:21.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:21.348 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:21.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:21.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:21.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:21.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:21.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:21.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:21.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:01:21.734 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:21.737 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:21.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:21.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:21.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:21.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:21.909 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:21.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:21.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:21.982 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 43013 42997 149.265 94.8125 0.00236527 0.00661482 2024-09-13T15:01:21.988 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[113926]: pgmap v1260: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T15:01:21.989 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:21.989 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:21.989 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[108570]: pgmap v1260: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T15:01:21.989 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:21.989 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:22.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:22.034 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:22.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:22.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:21 smithi159 ceph-mon[102519]: pgmap v1260: 99 pgs: 99 active+clean; 2.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-09-13T15:01:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:01:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:22.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:22.366 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:22.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:22.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:22.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:22.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:22.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:22.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:22.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:22.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:22.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:22.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:22.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:22.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:22.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 44199 44183 145.306 74.125 0.00333438 0.00687721 2024-09-13T15:01:23.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:23.085 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:23.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:23.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:23.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:23.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:23.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:23.448 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:23.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:23.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:23.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:23.625 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:23.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:23.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:23.762 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:23.806 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:01:23.863 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:23.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:23.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:23.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:23.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:01:23.982801+0000 min lat: 0.000998522 max lat: 0.607686 avg lat: 0.00695841 2024-09-13T15:01:23.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:01:23.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 45982 45966 143.612 111.438 0.00738723 0.00695841 2024-09-13T15:01:24.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:24.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:24.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:23 smithi042 ceph-mon[113926]: pgmap v1261: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:01:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:23 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:23 smithi042 ceph-mon[108570]: pgmap v1261: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:01:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:23 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:24.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:23 smithi159 ceph-mon[102519]: pgmap v1261: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:01:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:23 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:24.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:24.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:24.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:24.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:24.457 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:24.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:24.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:24.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:24.699 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:24.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:24.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:24.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:24.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:24.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:24.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:24.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 47839 47823 142.3 116.062 0.00592682 0.00702197 2024-09-13T15:01:25.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:25.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:25.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:25.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:25.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[108570]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T15:01:25.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:25.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:25.103 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:24 smithi042 ceph-mon[113926]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T15:01:25.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:25.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:24 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:24 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2440094097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:24 smithi159 ceph-mon[102519]: osdmap e353: 8 total, 8 up, 8 in 2024-09-13T15:01:25.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:25.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:25.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:25.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:25.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:25.570 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:25.672 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:25.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:25.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:25.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:25.953 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:25.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 49577 49561 140.769 108.625 0.0045076 0.00709874 2024-09-13T15:01:25.986 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:25.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:25.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:25.998 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:26.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:25 smithi042 ceph-mon[108570]: pgmap v1263: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:01:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:25 smithi042 ceph-mon[113926]: pgmap v1263: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:01:26.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:26.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:25 smithi159 ceph-mon[102519]: pgmap v1263: 99 pgs: 99 active+clean; 2.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-09-13T15:01:26.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:26.359 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:26.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.534 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:26.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:26.674 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:26.691 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:01:26.691 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:26.731 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:01:26.731 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:26.768 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:01:26.769 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:26.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:26.805 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:01:26.805 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:26.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:26.850 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:01:26.851 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:26.892 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:01:26.892 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:26.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:26.920 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:01:26.920 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:26.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:26.938 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:01:26.938 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:26.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:26.957 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:26 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1009569012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:26.957 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:26 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1009569012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:26.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 51100 51084 138.787 95.1875 0.0101689 0.00720023 2024-09-13T15:01:27.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:27.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:27.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:27.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:27.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:27.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:26 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1009569012' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:27.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:27.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:27.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:27.471 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:27.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:27.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:27.595 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:01:27.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:27.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:27.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:27.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:27.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:27.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:27.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:27.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:27.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:27.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 52664 52648 137.077 97.75 0.009126 0.00729022 2024-09-13T15:01:28.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:28.102 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:27 smithi042 ceph-mon[108570]: pgmap v1264: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T15:01:28.102 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:27 smithi042 ceph-mon[113926]: pgmap v1264: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T15:01:28.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:28.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:27 smithi159 ceph-mon[102519]: pgmap v1264: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-09-13T15:01:28.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.381 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:28.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:28.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:28.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:28.661 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:28.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:28.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:28.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:28.934 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:28.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:28.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:28.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 54317 54301 135.727 103.312 0.00886241 0.00736266 2024-09-13T15:01:29.036 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:29.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:29.334 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:29.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:29.436 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:29.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:29.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:29.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:29.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:29.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:29.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:29.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:29.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:29.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:29.968 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:29.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 55567 55551 133.511 78.125 0.00783945 0.00748455 2024-09-13T15:01:30.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:29 smithi042 ceph-mon[108570]: pgmap v1265: 99 pgs: 99 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:01:30.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:29 smithi042 ceph-mon[113926]: pgmap v1265: 99 pgs: 99 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:01:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:29] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.51.0" 2024-09-13T15:01:30.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:30.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:30.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:30.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:29 smithi159 ceph-mon[102519]: pgmap v1265: 99 pgs: 99 active+clean; 3.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2024-09-13T15:01:30.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:30.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:30.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:30.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:30.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:30.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:30.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:30.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:30.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:30.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:30.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:30.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 57206 57190 132.36 102.438 0.00632999 0.00755052 2024-09-13T15:01:31.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:31.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:31.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:31.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:31.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:31.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:31.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:31.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:31.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:31.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:31.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:31.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:31.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:31.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:31.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:31.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:31.938 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:31.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:31.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:31.983 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 58803 58787 131.197 99.8125 0.023298 0.00761646 2024-09-13T15:01:32.040 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:32.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:31 smithi042 ceph-mon[113926]: pgmap v1266: 99 pgs: 99 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T15:01:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:31 smithi042 ceph-mon[108570]: pgmap v1266: 99 pgs: 99 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T15:01:32.177 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:32.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:31 smithi159 ceph-mon[102519]: pgmap v1266: 99 pgs: 99 active+clean; 3.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-13T15:01:32.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:32.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:32.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:32.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:32.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:32.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.627 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:32.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:32.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:32.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:32.841 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.5 primary-affinity to 0 (802) 2024-09-13T15:01:32.859 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-13T15:01:32.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:32.897 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:32.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:32.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:32.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 60204 60188 129.692 87.5625 0.00283605 0.00767731 2024-09-13T15:01:32.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:33.021 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:33.116 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-13T15:01:33.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:33.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:33.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:33.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 0.0} : dispatch 2024-09-13T15:01:33.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:33.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.393 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:33.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:33.495 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:33.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:33.620 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:33.641 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:33.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:33.764 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:33.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:33.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:33.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:33.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:33.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 60402 60386 125.782 12.375 0.01317 0.00794573 2024-09-13T15:01:34.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:34.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:34.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[113926]: pgmap v1267: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T15:01:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[113926]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[113926]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[108570]: pgmap v1267: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[108570]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T15:01:34.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:33 smithi042 ceph-mon[108570]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T15:01:34.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:34.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:33 smithi159 ceph-mon[102519]: pgmap v1267: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-09-13T15:01:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3859586387' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.0}]': finished 2024-09-13T15:01:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:33 smithi159 ceph-mon[102519]: osdmap e354: 8 total, 8 up, 8 in 2024-09-13T15:01:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:33 smithi159 ceph-mon[102519]: osdmap e355: 8 total, 8 up, 8 in 2024-09-13T15:01:34.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:34.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:34.373 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:34.411 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:01:34.411 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:01:34.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:01:34.411 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:01:34.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86125 12.11 12:8a0d8b36:::benchmark_data_smithi042_260592_object86124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 222.255756131, 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3037125899999999, 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.412 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.413 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86125 2024-09-13T15:01:34.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.413 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:01:34.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3399999999999999e-07 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151505+0000", 2024-09-13T15:01:34.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0979e-05 2024-09-13T15:01:34.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151513+0000", 2024-09-13T15:01:34.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3180000000000001e-06 2024-09-13T15:01:34.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151518+0000", 2024-09-13T15:01:34.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4040000000000003e-06 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152398+0000", 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00087995399999999996 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152404+0000", 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1680000000000001e-06 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416477+0000", 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2640725930000001 2024-09-13T15:01:34.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416584+0000", 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010674 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416736+0000", 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015215300000000001 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422773+0000", 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060373429999999997 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445727+0000", 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022953881999999998 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453059+0000", 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0073320290000000003 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453124+0000", 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5162999999999999e-05 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.455196+0000", 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020723299999999998 2024-09-13T15:01:34.422 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86135 12.11 12:8c7021b7:::benchmark_data_smithi042_260592_object86134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 222.25571512299999, 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3064091740000001, 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.423 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86135 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4000000000000002e-07 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151544+0000", 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8833000000000001e-05 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151551+0000", 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7279999999999998e-06 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151556+0000", 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3129999999999998e-06 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152425+0000", 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00086871300000000002 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152431+0000", 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.144e-06 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.417930+0000", 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2654990859999999 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.428 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418028+0000", 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7684999999999997e-05 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418152+0000", 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012413100000000001 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.429 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422904+0000", 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047515930000000001 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445774+0000", 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022870715 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.430 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456350+0000", 2024-09-13T15:01:34.431 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010575721 2024-09-13T15:01:34.431 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.431 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.431 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.431 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:34.432 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456409+0000", 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9438000000000002e-05 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.457934+0000", 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001524634 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86235 12.11 12:8a3e8959:::benchmark_data_smithi042_260592_object86234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 222.25567762200001, 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3274169330000001, 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.434 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86235 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.435 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151563+0000", 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2300000000000002e-07 2024-09-13T15:01:34.436 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151587+0000", 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3813000000000001e-05 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151593+0000", 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.844e-06 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.437 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151599+0000", 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2009999999999998e-06 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152452+0000", 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085325899999999998 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.438 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152458+0000", 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.049e-06 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422991+0000", 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.270532545 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.439 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423159+0000", 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016851999999999999 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423348+0000", 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018931899999999999 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.428067+0000", 2024-09-13T15:01:34.440 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047184879999999998 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445827+0000", 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017759664000000001 2024-09-13T15:01:34.441 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474222+0000", 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.028395482999999999 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.442 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.443 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.443 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474279+0000", 2024-09-13T15:01:34.443 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7204999999999999e-05 2024-09-13T15:01:34.443 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.443 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.478979+0000", 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047001200000000003 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.444 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86275 12.11 12:8d556199:::benchmark_data_smithi042_260592_object86274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 222.25563505100001, 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3296465079999999, 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86275 2024-09-13T15:01:34.445 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.446 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4999999999999998e-07 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151627+0000", 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1914999999999999e-05 2024-09-13T15:01:34.448 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151634+0000", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6479999999999997e-06 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151639+0000", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.578e-06 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152479+0000", 2024-09-13T15:01:34.449 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00084047799999999997 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152485+0000", 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.207e-06 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424679+0000", 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721941349999999 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.450 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424837+0000", 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015725900000000001 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.425047+0000", 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000210137 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.451 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.429698+0000", 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046507639999999999 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445901+0000", 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016203077999999999 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479037+0000", 2024-09-13T15:01:34.452 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033136456000000002 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479191+0000", 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015432 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.481251+0000", 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020600829999999999 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.453 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:214128 12.2 12:475b0344:::benchmark_data_smithi042_260592_object97237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:59:59.551296+0000", 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 94.855944070000007, 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.14224198199999999, 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.454 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214128 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551296+0000", 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551296+0000", 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.455 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551297+0000", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.108e-06 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551302+0000", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7480000000000001e-06 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551314+0000", 2024-09-13T15:01:34.456 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2133999999999999e-05 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.551322+0000", 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6869999999999994e-06 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.552762+0000", 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014403790000000001 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.552907+0000", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014547099999999999 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.553501+0000", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00059317899999999999 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.559924+0000", 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064234979999999997 2024-09-13T15:01:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.562554+0000", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026295329999999999 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.566879+0000", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0043255450000000001 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567229+0000", 2024-09-13T15:01:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034943499999999999 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.693538+0000", 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.126309265 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:214132 12.18 12:1eb94477:::benchmark_data_smithi042_260592_object97241:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-09-13T15:01:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:59:59.557256+0000", 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 94.849984352999996, 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.140768958, 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214132 2024-09-13T15:01:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557256+0000", 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557256+0000", 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557257+0000", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6869999999999999e-06 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557263+0000", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8000000000000004e-06 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557274+0000", 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1206000000000001e-05 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557282+0000", 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5240000000000003e-06 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.559439+0000", 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002156842 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.559544+0000", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000104864 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.560046+0000", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00050206899999999995 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.562761+0000", 2024-09-13T15:01:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027157230000000002 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.697848+0000", 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13508661799999999 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.697915+0000", 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6623000000000002e-05 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.698025+0000", 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011000199999999999 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:214134 12.e 12:71ce9c7c:::benchmark_data_smithi042_260592_object97243:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:59:59.557304+0000", 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 94.849935943000006, 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.135495907, 2024-09-13T15:01:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214134 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557304+0000", 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557304+0000", 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557305+0000", 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7300000000000001e-07 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557307+0000", 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2290000000000002e-06 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557313+0000", 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3620000000000002e-06 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557318+0000", 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6979999999999997e-06 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557373+0000", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5138e-05 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557509+0000", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013647 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.557633+0000", 2024-09-13T15:01:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012322799999999999 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.562490+0000", 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048568500000000002 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.566533+0000", 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040433539999999999 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.692623+0000", 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12609032000000001 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.692687+0000", 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4305000000000002e-05 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.692800+0000", 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011247999999999999 2024-09-13T15:01:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:214144 12.18 12:1b803406:::benchmark_data_smithi042_260592_object97253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:59:59.567468+0000", 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 94.839772081000007, 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13851913499999999, 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214144 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567468+0000", 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567468+0000", 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567469+0000", 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.429e-06 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567474+0000", 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.668e-06 2024-09-13T15:01:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567486+0000", 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1659e-05 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.567497+0000", 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1109e-05 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.692484+0000", 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12498753 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.692681+0000", 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000196994 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.693290+0000", 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00060825199999999999 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.698042+0000", 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047529249999999999 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.705803+0000", 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0077601190000000002 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.705911+0000", 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010851 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:59:59.705987+0000", 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5939999999999995e-05 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:217198 12.18 12:1b470e9a:::benchmark_data_smithi042_260592_object100307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.975225+0000", 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 91.432014936000002, 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.14442540800000001, 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217198 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975225+0000", 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975225+0000", 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975227+0000", 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.793e-06 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975233+0000", 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.677e-06 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975246+0000", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3001e-05 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975253+0000", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5639999999999999e-06 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975314+0000", 2024-09-13T15:01:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1049000000000005e-05 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.975441+0000", 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012718699999999999 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2", 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.976008+0000", 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00056641500000000002 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.983892+0000", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078838999999999992 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119245+0000", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.13535344699999999 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119557+0000", 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000312107 2024-09-13T15:01:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119650+0000", 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3268000000000002e-05 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:217201 12.2 12:4755d4bb:::benchmark_data_smithi042_260592_object100310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 91.422632245000003, 2024-09-13T15:01:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.144661345, 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217201 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:01:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984610+0000", 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1210000000000001e-06 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984616+0000", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0839999999999997e-06 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984628+0000", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2517000000000001e-05 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984636+0000", 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6699999999999994e-06 2024-09-13T15:01:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985073+0000", 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00043719400000000002 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985220+0000", 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014707300000000001 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:01:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985790+0000", 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00056957299999999998 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.123004+0000", 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.137214313 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126034+0000", 2024-09-13T15:01:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030292850000000001 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126049+0000", 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5109999999999999e-05 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.127662+0000", 2024-09-13T15:01:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001613225 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.129269+0000", 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016071799999999999 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:217205 12.1 e335/307 12:80358a7c:::benchmark_data_smithi042_260592_object100314:head v 335'9274, mlcod=335'9272)", 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.984680+0000", 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 91.422560027000003, 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.141780817, 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984680+0000", 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984680+0000", 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984681+0000", 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.71e-07 2024-09-13T15:01:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984687+0000", 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8649999999999998e-06 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984698+0000", 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1503e-05 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984705+0000", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0960000000000003e-06 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119084+0000", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.134378421 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119111+0000", 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6869999999999999e-05 2024-09-13T15:01:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126352+0000", 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0072416429999999999 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126461+0000", 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108548 2024-09-13T15:01:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:217208 12.b 12:d7d609ef:::benchmark_data_smithi042_260592_object100317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.987036+0000", 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 91.420203806999993, 2024-09-13T15:01:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.137003966, 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217208 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987036+0000", 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987036+0000", 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987038+0000", 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.404e-06 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987043+0000", 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8309999999999997e-06 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119049+0000", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.132005763 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119064+0000", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4579000000000001e-05 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119298+0000", 2024-09-13T15:01:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023458600000000001 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119458+0000", 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015971900000000001 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119641+0000", 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018277500000000001 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.122876+0000", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003234836 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.122958+0000", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2618000000000001e-05 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.123936+0000", 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00097777900000000002 2024-09-13T15:01:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.123993+0000", 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7083999999999998e-05 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.124040+0000", 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6992e-05 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45599.0:217207 12.19 e335/307 12:986456c0:::benchmark_data_smithi042_260592_object100316:head v 335'9235, mlcod=335'9231)", 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.987708+0000", 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 91.419531582000005, 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.142400797, 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987708+0000", 2024-09-13T15:01:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987708+0000", 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987709+0000", 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5499999999999996e-07 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.987716+0000", 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.511e-06 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119053+0000", 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.131336921 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119066+0000", 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3249e-05 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119135+0000", 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8829000000000001e-05 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.119147+0000", 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2221e-05 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.130055+0000", 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010907762 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.130109+0000", 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4348999999999998e-05 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:55028 13.b e353/344 13:da56cd9d:::benchmark_data_smithi042_286876_object55027:head v 353'3406, mlcod=353'3404)", 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4.9736586950000001, 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25938409099999998, 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433582+0000", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6000000000000002e-07 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433693+0000", 2024-09-13T15:01:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011040300000000001 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433704+0000", 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1885999999999999e-05 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687498+0000", 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25379304400000002 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687603+0000", 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105482 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687629+0000", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6492e-05 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692925+0000", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052956619999999996 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692965+0000", 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0262000000000002e-05 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:55062 13.6 e353/344 13:629fd30b:::benchmark_data_smithi042_286876_object55061:head v 353'3438, mlcod=353'3436)", 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4.9527187269999997, 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23819004399999999, 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454531+0000", 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0142999999999999e-05 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454596+0000", 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4306000000000004e-05 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687499+0000", 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.232902998 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687516+0000", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6940999999999999e-05 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687709+0000", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019348499999999999 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687727+0000", 2024-09-13T15:01:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7561e-05 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692674+0000", 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0049475880000000002 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692711+0000", 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7021999999999999e-05 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60162 13.2 13:418d4a79:::benchmark_data_smithi042_286876_object60161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.5417545270000002, 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0028493329999999, 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60162 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865487+0000", 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8250000000000001e-06 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865508+0000", 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0947000000000001e-05 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865524+0000", 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6052000000000001e-05 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865532+0000", 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5989999999999996e-06 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865587+0000", 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5349999999999997e-05 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865596+0000", 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9719999999999998e-06 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.859408+0000", 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99381218500000001 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860985+0000", 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001576446 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.861455+0000", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00047016700000000001 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864756+0000", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003301114 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865824+0000", 2024-09-13T15:01:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010678669999999999 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868257+0000", 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024326009999999999 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868292+0000", 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5324000000000003e-05 2024-09-13T15:01:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868335+0000", 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2883999999999998e-05 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60176 13.b 13:d1c41892:::benchmark_data_smithi042_286876_object60175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:01:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.5398594289999998, 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99908001400000002, 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60176 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867382+0000", 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5939999999999999e-06 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867408+0000", 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5731000000000001e-05 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867424+0000", 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6475000000000001e-05 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867435+0000", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0183999999999999e-05 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867533+0000", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8406999999999995e-05 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867544+0000", 2024-09-13T15:01:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.099e-05 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.858796+0000", 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99125208099999995 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860588+0000", 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001791988 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860723+0000", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013452399999999999 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866300+0000", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0055779109999999996 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866355+0000", 2024-09-13T15:01:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4098999999999998e-05 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866368+0000", 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3747999999999999e-05 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866419+0000", 2024-09-13T15:01:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0760999999999997e-05 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866461+0000", 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1520999999999998e-05 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60179 13.b 13:da237103:::benchmark_data_smithi042_286876_object60178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:01:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.5360495279999999, 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.004732119, 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60179 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871192+0000", 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.708e-06 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871215+0000", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2867e-05 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871233+0000", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7872000000000001e-05 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871244+0000", 2024-09-13T15:01:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1304000000000001e-05 2024-09-13T15:01:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871328+0000", 2024-09-13T15:01:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3912999999999994e-05 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871348+0000", 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9386e-05 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863786+0000", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99243854099999995 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863945+0000", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015913099999999999 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864079+0000", 2024-09-13T15:01:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013408800000000001 2024-09-13T15:01:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872431+0000", 2024-09-13T15:01:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0083515080000000005 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872656+0000", 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000225345 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875824+0000", 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031679709999999999 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875879+0000", 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4401000000000003e-05 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875923+0000", 2024-09-13T15:01:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4084e-05 2024-09-13T15:01:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60186 13.b 13:db94921c:::benchmark_data_smithi042_286876_object60185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.5325416139999999, 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.001358341, 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60186 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874700+0000", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.782e-06 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874727+0000", 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6758e-05 2024-09-13T15:01:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874746+0000", 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8569000000000001e-05 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874754+0000", 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1769999999999998e-06 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875021+0000", 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026734699999999997 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875031+0000", 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0098999999999999e-05 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866528+0000", 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.991496618 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866700+0000", 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017193800000000001 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866831+0000", 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013130999999999999 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872614+0000", 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057832559999999996 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875962+0000", 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003347629 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875974+0000", 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2177999999999999e-05 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876022+0000", 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8270999999999999e-05 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876057+0000", 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4409000000000002e-05 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60204 13.b 13:d5caf6b7:::benchmark_data_smithi042_286876_object60203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.5181637569999999, 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98712613199999999, 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60204 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:01:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889079+0000", 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.39e-06 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889102+0000", 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3379000000000001e-05 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889121+0000", 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8493999999999999e-05 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889131+0000", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0762e-05 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889195+0000", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3629999999999999e-05 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889202+0000", 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5889999999999997e-06 2024-09-13T15:01:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868745+0000", 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97954200499999999 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868871+0000", 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012671000000000001 2024-09-13T15:01:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869018+0000", 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014694099999999999 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872700+0000", 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003681658 2024-09-13T15:01:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876064+0000", 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033644360000000002 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876097+0000", 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3073e-05 2024-09-13T15:01:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876158+0000", 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0590000000000001e-05 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876202+0000", 2024-09-13T15:01:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4474999999999997e-05 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:01:34.568 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:01:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:34.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:34.670 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:34.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:34.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:34.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:34.849 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:34.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:34.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:34.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 61947 61931 124.839 96.5625 0.00609782 0.0080054 2024-09-13T15:01:34.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:34.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:35.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:35.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:35.139 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:35.230 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:35.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:35.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.411 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:35.513 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:35.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:35.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:35.691 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:35.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:35.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:35.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:35.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:35.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:35.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:35.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:35.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 63404 63388 123.784 91.0625 0.0059874 0.00807355 2024-09-13T15:01:36.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:36.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:36.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:36.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:36.241 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:35 smithi159 ceph-mon[102519]: pgmap v1270: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T15:01:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:35 smithi159 ceph-mon[102519]: 1.8 scrub starts 2024-09-13T15:01:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[108570]: pgmap v1270: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T15:01:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[108570]: 1.8 scrub starts 2024-09-13T15:01:36.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[113926]: pgmap v1270: 99 pgs: 99 active+clean; 3.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-13T15:01:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[113926]: 1.8 scrub starts 2024-09-13T15:01:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:36.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:36.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:36.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:36.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:36.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:36.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:36.654 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:36.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:36.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:36.794 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:36.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:36.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:36.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:01:36.934 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:36.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:36.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:36.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 64904 64888 122.873 93.75 0.00867761 0.00813336 2024-09-13T15:01:37.036 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:37.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:37.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:37.118 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:36 smithi042 ceph-mon[108570]: 1.a scrub starts 2024-09-13T15:01:37.119 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:36 smithi042 ceph-mon[113926]: 1.a scrub starts 2024-09-13T15:01:37.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:37.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:37.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:37.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:36 smithi159 ceph-mon[102519]: 1.a scrub starts 2024-09-13T15:01:37.314 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:37.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:37.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:37.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:37.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:37.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:37.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:37.593 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:37.633 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:37.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:37.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:37.767 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:37.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:37.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:37.905 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:37.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:37.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 66309 66293 121.842 87.8125 0.0104563 0.00820202 2024-09-13T15:01:38.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.041 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:38.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:38.084 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:37 smithi159 ceph-mon[102519]: pgmap v1271: 99 pgs: 99 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:01:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:37 smithi042 ceph-mon[108570]: pgmap v1271: 99 pgs: 99 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:01:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:37 smithi042 ceph-mon[113926]: pgmap v1271: 99 pgs: 99 active+clean; 3.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:01:38.143 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:38.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:38.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:38.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:38.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:38.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:38.559 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:38.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:38.599 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:38.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:38.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:38.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:38.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:38.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:38.880 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:38.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:38.981 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:38.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 67660 67644 120.773 84.4375 0.00554914 0.00827413 2024-09-13T15:01:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:39.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:39.121 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:39.133 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 0, 4] out_osds: [] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:01:39.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:01:39.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:39.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:39.261 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:39.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:39.402 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:39.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:39.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:39.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:39.592 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:39.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:39.749 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:39.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:39.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:39.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:39.900 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:01:39.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:39 smithi042 ceph-mon[108570]: pgmap v1272: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 103 MiB/s wr, 1.66k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:01:39.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:39.901 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:39] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T15:01:39.901 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:39 smithi042 ceph-mon[113926]: pgmap v1272: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 103 MiB/s wr, 1.66k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:01:39.901 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:39.984 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 69080 69064 119.884 88.75 0.0170258 0.00833614 2024-09-13T15:01:39.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:40.032 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:40.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:40.154 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:39 smithi159 ceph-mon[102519]: pgmap v1272: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 103 MiB/s wr, 1.66k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:01:40.154 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:01:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:40.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:40.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69486 13.5 13:af9000de:::benchmark_data_smithi042_286876_object69485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.291878+0000", 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.026009726, 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026045876999999999, 2024-09-13T15:01:40.321 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69486 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291878+0000", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.322 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291878+0000", 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291877+0000", 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291906+0000", 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8875000000000001e-05 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291914+0000", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3470000000000001e-06 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291919+0000", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4190000000000001e-06 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.294622+0000", 2024-09-13T15:01:40.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027024950000000001 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.294744+0000", 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121994 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:40.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.294860+0000", 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000116012 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.297479+0000", 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026192749999999999 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.301503+0000", 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0040241950000000004 2024-09-13T15:01:40.326 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:69509 13.a e356/344 13:5c37e05a:::benchmark_data_smithi042_286876_object69508:head v 356'4308, mlcod=356'4306)", 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.315181+0000", 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.002706502, 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028414410000000001, 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315181+0000", 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315181+0000", 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315180+0000", 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:40.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315219+0000", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8186000000000003e-05 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315229+0000", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0197000000000001e-05 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315235+0000", 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9240000000000004e-06 2024-09-13T15:01:40.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315566+0000", 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00033128500000000001 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.315585+0000", 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9341e-05 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.330 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69507 13.d e356/344)", 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.317778+0000", 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00010991400000000001, 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00030333299999999998, 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317778+0000", 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317778+0000", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317777+0000", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317783+0000", 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0920000000000003e-06 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317799+0000", 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6489000000000001e-05 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.317807+0000", 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5809999999999999e-06 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.333 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69507 13.d e356/344)", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.318063+0000", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4294967295.9998245, 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0312999999999997e-05, 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318063+0000", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318063+0000", 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318062+0000", 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318067+0000", 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9520000000000002e-06 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318078+0000", 2024-09-13T15:01:40.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1018e-05 2024-09-13T15:01:40.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318084+0000", 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6930000000000004e-06 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69495 13.5 13:ac7c87b6:::benchmark_data_smithi042_286876_object69494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.298645+0000", 2024-09-13T15:01:40.337 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.019242622000000001, 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019533620000000002, 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69495 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298645+0000", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298645+0000", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298644+0000", 2024-09-13T15:01:40.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298678+0000", 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3988000000000001e-05 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298690+0000", 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2126999999999999e-05 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298697+0000", 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3200000000000002e-06 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298814+0000", 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000116521 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.298979+0000", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016494900000000001 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.299106+0000", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012747799999999999 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.304013+0000", 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0049063090000000002 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305244+0000", 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0012317490000000001 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69502 13.5 13:aab5b997:::benchmark_data_smithi042_286876_object69501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.305721+0000", 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.01216683, 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012553172, 2024-09-13T15:01:40.343 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69502 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305721+0000", 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305721+0000", 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305719+0000", 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305758+0000", 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8770000000000003e-05 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305772+0000", 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4191e-05 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.346 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305779+0000", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0330000000000003e-06 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.305813+0000", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3300999999999998e-05 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.306010+0000", 2024-09-13T15:01:40.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000197307 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.306177+0000", 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000167372 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:01:40.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310048+0000", 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038710929999999999 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312879+0000", 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00283044 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69507 13.d 13:b5f62f67:::benchmark_data_smithi042_286876_object69506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.309993+0000", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0078951000000000004, 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0083709240000000001, 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69507 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.309993+0000", 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.309993+0000", 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.309991+0000", 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:40.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310034+0000", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3129e-05 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310046+0000", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1343e-05 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310052+0000", 2024-09-13T15:01:40.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7460000000000003e-06 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312133+0000", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0020804159999999999 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312295+0000", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016239800000000001 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312431+0000", 2024-09-13T15:01:40.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000135415 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69470 13.5 13:ab8a4ffc:::benchmark_data_smithi042_286876_object69469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.279434+0000", 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.038453326000000003, 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.039010027000000003, 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.354 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69470 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279434+0000", 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279434+0000", 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279433+0000", 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279472+0000", 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9047000000000001e-05 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279481+0000", 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8720000000000008e-06 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279486+0000", 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1189999999999996e-06 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279602+0000", 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000115741 2024-09-13T15:01:40.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279758+0000", 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000156284 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.279893+0000", 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013479 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.287654+0000", 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0077616070000000002 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291652+0000", 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039980559999999998 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69508 13.d 13.7af3a93d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.310072+0000", 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0078159709999999997, 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0084575520000000001, 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69508 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.360 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310072+0000", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310072+0000", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310071+0000", 2024-09-13T15:01:40.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310111+0000", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9671e-05 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310121+0000", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0138e-05 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.310128+0000", 2024-09-13T15:01:40.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7689999999999999e-06 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69510 13.5 13.86fea735 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.312802+0000", 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.005085722, 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057802690000000002, 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.363 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69510 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312802+0000", 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312802+0000", 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312801+0000", 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312905+0000", 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000104595 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312914+0000", 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5439999999999999e-06 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.312919+0000", 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4689999999999996e-06 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.367 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69470 13.5 e356/348 ondisk, result = 0)", 2024-09-13T15:01:40.367 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.314399+0000", 2024-09-13T15:01:40.367 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0034889249999999999, 2024-09-13T15:01:40.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.368 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:40.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042344899999999996, 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314399+0000", 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314399+0000", 2024-09-13T15:01:40.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314398+0000", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314404+0000", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.711e-06 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314420+0000", 2024-09-13T15:01:40.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6541999999999998e-05 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314428+0000", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5680000000000002e-06 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318609+0000", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041809990000000003 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.318617+0000", 2024-09-13T15:01:40.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9279999999999993e-06 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69483 13.5 e356/348)", 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.314434+0000", 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0034538189999999999, 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042542459999999997, 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314434+0000", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314434+0000", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314434+0000", 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314436+0000", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2749999999999998e-06 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314441+0000", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8409999999999999e-06 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314444+0000", 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.788e-06 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.374 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:69483 13.5 13:a8f139e8:::benchmark_data_smithi042_286876_object69482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e356)", 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.291738+0000", 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.026149806000000001, 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027003659999999999, 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 69483 2024-09-13T15:01:40.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291738+0000", 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291738+0000", 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291736+0000", 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291783+0000", 2024-09-13T15:01:40.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7132000000000002e-05 2024-09-13T15:01:40.381 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.381 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.381 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.381 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291800+0000", 2024-09-13T15:01:40.381 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7275e-05 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291809+0000", 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8759999999999994e-06 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.382 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.291957+0000", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014805800000000001 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.292137+0000", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017946999999999999 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 0,5", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.292295+0000", 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015842699999999999 2024-09-13T15:01:40.383 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.297017+0000", 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047221190000000003 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.297659+0000", 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00064145399999999996 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.384 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69486 13.5 e356/348)", 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.314448+0000", 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0034396600000000002, 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004375332, 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314448+0000", 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.385 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314448+0000", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314448+0000", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314449+0000", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6959999999999999e-06 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314454+0000", 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4610000000000001e-06 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314457+0000", 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2260000000000001e-06 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.387 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:69495 13.5 e356/348)", 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:40.314460+0000", 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0034275170000000002, 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044151490000000002, 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:01:40.388 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314460+0000", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314460+0000", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314460+0000", 2024-09-13T15:01:40.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314462+0000", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.53e-06 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314466+0000", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.33e-06 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:40.314469+0000", 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3289999999999998e-06 2024-09-13T15:01:40.390 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 15 2024-09-13T15:01:40.391 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:01:40.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.507 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:40.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:40.609 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:40.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:40.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:40.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:40.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:40.919 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:40.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:40.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:40.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 70404 70388 118.88 82.75 0.00349471 0.00840293 2024-09-13T15:01:41.020 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.064 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:41.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:41.134 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:41.134 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:41.134 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[113926]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T15:01:41.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:41.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:41.135 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:40 smithi042 ceph-mon[108570]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T15:01:41.164 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:41.196 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:41.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:41.204 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:40 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:01:41.204 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1080076880' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:01:41.204 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:40 smithi159 ceph-mon[102519]: osdmap e356: 8 total, 8 up, 8 in 2024-09-13T15:01:41.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:01:41.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:41.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:41.466 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:41.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:41.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:41.633 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:41.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:41.650 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:41.735 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:41.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:41.875 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:41.916 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:41.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:41.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 71742 71726 117.952 83.625 0.0133196 0.00847255 2024-09-13T15:01:41.994 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:01:41.994 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:01:41.994 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:41.995 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:41.996 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:41.996 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:41.996 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:41.996 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:42.018 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:42.051 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:42.153 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:42.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:41 smithi159 ceph-mon[102519]: pgmap v1274: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 59 MiB/s wr, 957 op/s; 40 KiB/s, 0 objects/s recovering 2024-09-13T15:01:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:41 smithi159 ceph-mon[102519]: 1.8 scrub starts 2024-09-13T15:01:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:41 smithi159 ceph-mon[102519]: 1.8 scrub ok 2024-09-13T15:01:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1236859526' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:42.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:42.313 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[108570]: pgmap v1274: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 59 MiB/s wr, 957 op/s; 40 KiB/s, 0 objects/s recovering 2024-09-13T15:01:42.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[108570]: 1.8 scrub starts 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[108570]: 1.8 scrub ok 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1236859526' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[113926]: pgmap v1274: 99 pgs: 99 active+clean; 3.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 19 KiB/s rd, 59 MiB/s wr, 957 op/s; 40 KiB/s, 0 objects/s recovering 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[113926]: 1.8 scrub starts 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[113926]: 1.8 scrub ok 2024-09-13T15:01:42.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1236859526' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:42.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:42.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:42.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:42.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:42.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:42.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:42.792 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:42.818 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:42.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:42.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:42.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:42.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:42.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 72249 72233 115.74 31.6875 0.0110566 0.00863518 2024-09-13T15:01:43.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:43.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:43.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:43.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:43.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:43.209 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:42 smithi159 ceph-mon[102519]: 1.a scrub starts 2024-09-13T15:01:43.209 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:42 smithi159 ceph-mon[102519]: 1.a scrub ok 2024-09-13T15:01:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:43.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:43.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:42 smithi042 ceph-mon[113926]: 1.a scrub starts 2024-09-13T15:01:43.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:42 smithi042 ceph-mon[113926]: 1.a scrub ok 2024-09-13T15:01:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:42 smithi042 ceph-mon[108570]: 1.a scrub starts 2024-09-13T15:01:43.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:42 smithi042 ceph-mon[108570]: 1.a scrub ok 2024-09-13T15:01:43.351 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:43.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:43.480 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:43.503 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:43.583 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:43.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.616 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:43.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:43.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:43.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:43.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:43.849 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:43.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:43.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:43.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:43.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:01:43.984581+0000 min lat: 0.000998522 max lat: 1.01725 avg lat: 0.00873844 2024-09-13T15:01:43.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:01:43.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 73225 73209 114.372 61 0.00966553 0.00873844 2024-09-13T15:01:43.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.028 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:44.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:44.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:44.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:44.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:43 smithi159 ceph-mon[102519]: pgmap v1275: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T15:01:44.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:44.297 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:44.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:43 smithi042 ceph-mon[113926]: pgmap v1275: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T15:01:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:43 smithi042 ceph-mon[108570]: pgmap v1275: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s 2024-09-13T15:01:44.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:44.436 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:44.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:44.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:44.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:44.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:44.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:44.813 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:44.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:44.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:44.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:44.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 74549 74533 113.6 82.75 0.00959509 0.00879785 2024-09-13T15:01:44.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:44.986 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:44.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:45.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:45.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:45.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:45.268 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:45.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:45.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:45.407 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:45.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:45.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.552 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:45.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:45.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:45.699 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:45.785 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:45.785 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:45.801 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:45.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:45.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:45.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:45.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 75832 75816 112.804 80.1875 0.0125346 0.00885935 2024-09-13T15:01:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:45.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:45.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:46.119 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:46.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.185 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:45 smithi042 ceph-mon[108570]: pgmap v1276: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:01:46.186 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:45 smithi042 ceph-mon[113926]: pgmap v1276: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:01:46.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:45 smithi159 ceph-mon[102519]: pgmap v1276: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-09-13T15:01:46.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:46.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:46.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:46.399 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:01:46.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:46.533 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:46.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:46.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:46.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:46.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:46.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:46.818 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:46.846 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:46.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:01:46.948 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:46.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:46.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:46.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 76952 76936 111.809 70 0.00401711 0.00892428 2024-09-13T15:01:47.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:47.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:47.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:47.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:47.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:47.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:47.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:47.372 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:47.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:47.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:47.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:47.650 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:47.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:47.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:47.786 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:47.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:47.912 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:47.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:47.925 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:01:47.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 78097 78081 110.894 71.5625 0.00554275 0.00901237 2024-09-13T15:01:48.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:48.040 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:47 smithi042 ceph-mon[113926]: pgmap v1277: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:01:48.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:47 smithi042 ceph-mon[108570]: pgmap v1277: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:01:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:48.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:48.178 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:48.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:47 smithi159 ceph-mon[102519]: pgmap v1277: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:01:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:01:48.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:48.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:48.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:48.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:48.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.601 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:48.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:48.702 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:48.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:48.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:01:48.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:48.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:48.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:48.966 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:48.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 79226 79210 109.998 70.5625 0.00884637 0.00908553 2024-09-13T15:01:48.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:48.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:49.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:49.001 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:48 smithi159 ceph-mon[102519]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T15:01:49.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:49.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:48 smithi042 ceph-mon[113926]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T15:01:49.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4194451142' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:01:49.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:48 smithi042 ceph-mon[108570]: osdmap e357: 8 total, 8 up, 8 in 2024-09-13T15:01:49.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:49.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:49.168 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:49.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:01:49.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:49.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:49.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:49.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:49.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:49.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:49.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:49.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:49.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:49.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:49.735 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:49.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:49.858 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:49.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:49.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 80314 80298 109.085 68 0.0114767 0.00916213 2024-09-13T15:01:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:49.996 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:49.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.020 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:49 smithi042 ceph-mon[113926]: pgmap v1279: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:01:50.021 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:49 smithi042 ceph-mon[108570]: pgmap v1279: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:01:50.021 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:49] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T15:01:50.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:50.133 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:50.198 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:49 smithi159 ceph-mon[102519]: pgmap v1279: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-13T15:01:50.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:50.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:50.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:50.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:50.559 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:50.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:50.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:50.699 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:50.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:50.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:50.834 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:50.936 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:50.976 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:50.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:50.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:50.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 81563 81547 108.424 78.0625 0.00686461 0.00921741 2024-09-13T15:01:51.045 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.077 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:51.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:51.208 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:01:51.208 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:01:51.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:51.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:51.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:01:51.353 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:51.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:51.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:51.541 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:51.611 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:51.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:51.680 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:51.782 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:51.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:51.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:51.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:51.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:51.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 16 82725 82709 107.678 72.625 0.00392878 0.00926962 2024-09-13T15:01:52.010 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:01:52.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:52.078 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:51 smithi042 ceph-mon[113926]: pgmap v1280: 99 pgs: 99 active+clean; 4.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T15:01:52.079 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:52.079 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:51 smithi042 ceph-mon[108570]: pgmap v1280: 99 pgs: 99 active+clean; 4.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T15:01:52.079 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:52.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:52.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:52.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:52.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:51 smithi159 ceph-mon[102519]: pgmap v1280: 99 pgs: 99 active+clean; 4.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.20k op/s 2024-09-13T15:01:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:01:52.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:52.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:52.483 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:52.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:52.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:52.656 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:52.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:52.757 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:52.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:52.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:52.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:52.938 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:52.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:52.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:52.941 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:01:52.941 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:01:52.941 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:01:52.941 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:01:52.942 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:01:52.943 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:01:52.943 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:01:52.943 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:01:52.985 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 83922 83906 107.008 74.8125 0.0137963 0.00934033 2024-09-13T15:01:53.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:53.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:53.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:53.227 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[113926]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/907257131' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[108570]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T15:01:53.228 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/907257131' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:53.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:53.230 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:01:53.230 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:01:53.230 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1043180809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:01:53.230 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: osdmap e358: 8 total, 8 up, 8 in 2024-09-13T15:01:53.230 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/907257131' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:01:53.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:01:53.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:53.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:53.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:53.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:53.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:53.520 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:53.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:53.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:53.596 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:53.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:53.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:53.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:53.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.829 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:53.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:53.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:53.961 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:53.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:53.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:53.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 85118 85102 106.362 74.75 0.0137516 0.00939693 2024-09-13T15:01:54.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:54.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:54.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:54.269 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:54 smithi042 ceph-mon[113926]: pgmap v1282: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:01:54.270 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:54 smithi042 ceph-mon[108570]: pgmap v1282: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:01:54.340 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:54.380 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:54.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:54 smithi159 ceph-mon[102519]: pgmap v1282: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:01:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:54.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:54.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:54.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:54.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.800 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:54.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:01:54.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:54.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:54.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:54.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:54.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 86328 86312 105.76 75.625 0.00871611 0.00945044 2024-09-13T15:01:55.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:55.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:55.269 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:01:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:55.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:55.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:55.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:55.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:55.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:55.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:55.699 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:01:55.700 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:01:55.700 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T15:01:55.700 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:01:55.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:55.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:55.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:55.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:55.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:55.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:55.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 87348 87332 104.952 63.75 0.00644305 0.00952311 2024-09-13T15:01:56.064 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:01:56.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:56.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:56.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:56.240 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:56.316 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:56 smithi042 ceph-mon[108570]: pgmap v1283: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-09-13T15:01:56.316 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:56 smithi042 ceph-mon[113926]: pgmap v1283: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-09-13T15:01:56.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:56.375 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:56.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:56.391 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:56 smithi159 ceph-mon[102519]: pgmap v1283: 99 pgs: 99 active+clean; 5.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2024-09-13T15:01:56.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:01:56.477 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:56.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:56.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.709 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:56.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:56.794 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:01:56.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.854 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:56.956 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:56.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 88424 88408 104.24 67.25 0.0226503 0.00958866 2024-09-13T15:01:56.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:56.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:57.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:57.137 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:57.239 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:57.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:57.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:57.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:57.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:57.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:57.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:57.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:57.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:57.691 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:57.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:57.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:57.935 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:57.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:57.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:57.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 89581 89565 103.649 72.3125 0.00664925 0.0096429 2024-09-13T15:01:58.067 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:58 smithi042 ceph-mon[113926]: pgmap v1284: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:01:58.068 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:58.068 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:58 smithi042 ceph-mon[108570]: pgmap v1284: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:01:58.068 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:58.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:58.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:58.141 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:58 smithi159 ceph-mon[102519]: pgmap v1284: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-13T15:01:58.142 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:01:58.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:01:58.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.256 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:58.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:01:58.359 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:58.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:58.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:58.484 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:01:58.505 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:58.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:01:58.607 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:01:58.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:58.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:01:58.745 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:01:58.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:58.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:01:58.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:58.906 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:58.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:58.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 16 90347 90331 102.635 47.875 0.00846861 0.00973837 2024-09-13T15:01:59.007 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:01:59.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:01:59.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:59.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:59.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[113926]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T15:01:59.289 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:59.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:59.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[108570]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T15:01:59.290 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:59.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2956254671' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:01:59.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:59 smithi159 ceph-mon[102519]: osdmap e359: 8 total, 8 up, 8 in 2024-09-13T15:01:59.381 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:01:59 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:01:59.382 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:01:59.382 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:01:59.382 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:01:59.382 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:01:59.382 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:01:59.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:01:59.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:01:59.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:59.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:01:59.754 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:01:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:01:59.856 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:01:59.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:01:59.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:01:59.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:01:59.905 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:01:59.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 91520 91504 102.111 73.3125 0.0285656 0.00978781 2024-09-13T15:01:59.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:00.024 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:00.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:00.054 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:01:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:01:59] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-09-13T15:02:00.126 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:00.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:00.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:02:00.270 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:00.300 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:00.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:00.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:00 smithi042 ceph-mon[113926]: pgmap v1286: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:02:00.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:00 smithi042 ceph-mon[108570]: pgmap v1286: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:02:00.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.491 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:00.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:00 smithi159 ceph-mon[102519]: pgmap v1286: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-13T15:02:00.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:00.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.629 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:00.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:00.704 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:00.723 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T15:02:00.730 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:00.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:00.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:00.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:00.959 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:00.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:00.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 92548 92532 101.447 64.25 0.0252712 0.00985199 2024-09-13T15:02:01.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:01.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.102 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:01.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:01.203 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:01.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:01.258 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:01.353 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:01.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:01.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:01.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:01.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:01.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:01.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:01 smithi159 ceph-mon[102519]: pgmap v1287: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-13T15:02:01.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:01.825 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:01 smithi042 ceph-mon[113926]: pgmap v1287: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-13T15:02:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:01 smithi042 ceph-mon[108570]: pgmap v1287: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s 2024-09-13T15:02:01.927 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:01.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:01.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:01.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 93607 93591 100.839 66.1875 0.0269725 0.0099108 2024-09-13T15:02:02.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:02.100 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:02.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:02.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:02.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:02.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:02.343 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:02.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:02.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:02.488 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:02.516 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:02.564 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:02:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:02.618 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:02.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:02.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:02.801 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:02.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:02.831 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:02:02.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:02.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:02.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:02.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:02.986 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 94648 94632 100.232 65.0625 0.00664989 0.00997211 2024-09-13T15:02:03.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:03.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:03.195 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:03.297 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:03.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:03.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:03.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:03.499 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:03.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:03.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:03.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.647 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:03.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:03.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:03 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:03 smithi159 ceph-mon[102519]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T15:02:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:03 smithi159 ceph-mon[102519]: pgmap v1289: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T15:02:03.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[113926]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[113926]: pgmap v1289: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:03.805 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1657206209' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:02:03.806 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[108570]: osdmap e360: 8 total, 8 up, 8 in 2024-09-13T15:02:03.806 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:03 smithi042 ceph-mon[108570]: pgmap v1289: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-13T15:02:03.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:03.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:03.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:03.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:02:03.986394+0000 min lat: 0.000998522 max lat: 1.01725 avg lat: 0.0100367 2024-09-13T15:02:03.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:02:03.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 95627 95611 99.5816 61.1875 0.0176659 0.0100367 2024-09-13T15:02:04.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:04.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.056 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:04.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:04.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:04.197 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:04.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:04.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:04.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:04.430 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:04.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:04.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:04.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:04.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:04.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:04.715 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:04.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:04.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2655135168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:04.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2655135168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:04.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:04.889 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:02:04.889 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:02:04.889 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:02:04.889 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:04.890 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:02:04.891 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:02:04.891 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:02:04.891 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:04.891 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:04.898 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:04.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:04.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:04.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 15 96706 96691 99.0556 67.5 0.00839928 0.0100901 2024-09-13T15:02:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2655135168' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:05.000 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:05.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:05.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:05.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:05.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85746 12.11 12.52b0e931 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.49275073199999, 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73855728300000001, 2024-09-13T15:02:05.184 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85746 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.185 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412236+0000", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412398+0000", 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000161299 2024-09-13T15:02:05.186 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412410+0000", 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2771e-05 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412420+0000", 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6369999999999994e-06 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.187 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412466+0000", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5868999999999997e-05 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412472+0000", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6359999999999997e-06 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150786+0000", 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73831398199999998 2024-09-13T15:02:05.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150795+0000", 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7150000000000004e-06 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85758 12.11 12.1e200eb1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.482666798, 2024-09-13T15:02:05.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72850151299999999, 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85758 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422320+0000", 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422398+0000", 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7783999999999999e-05 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422410+0000", 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1781e-05 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422417+0000", 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5390000000000002e-06 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422459+0000", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1310000000000003e-05 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422468+0000", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1190000000000003e-06 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150816+0000", 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72834841400000006 2024-09-13T15:02:05.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150823+0000", 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0480000000000001e-06 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85838 12.11 12.30b94631 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.409152178, 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65501097900000005, 2024-09-13T15:02:05.194 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85838 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495834+0000", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495947+0000", 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000113403 2024-09-13T15:02:05.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495966+0000", 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.878e-05 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495975+0000", 2024-09-13T15:02:05.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2709999999999998e-06 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.496032+0000", 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6221999999999998e-05 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.496043+0000", 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1041e-05 2024-09-13T15:02:05.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150840+0000", 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65479739599999998 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150847+0000", 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3220000000000004e-06 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85847 12.11 12.a88885b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 252.39730288199999, 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64318668999999995, 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85847 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.200 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507684+0000", 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507727+0000", 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2503000000000003e-05 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507739+0000", 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.226e-05 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507745+0000", 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8370000000000002e-06 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507795+0000", 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9666999999999997e-05 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507802+0000", 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9270000000000005e-06 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150865+0000", 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64306261300000001 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.204 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150872+0000", 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3780000000000003e-06 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:91479 12.c 12:3e2fd148:::benchmark_data_smithi042_260592_object91478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 244.670220212, 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97834175000000001, 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.205 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 91479 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234767+0000", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234899+0000", 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013134 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234908+0000", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2399999999999996e-06 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234913+0000", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3929999999999999e-06 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234956+0000", 2024-09-13T15:02:05.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3319000000000003e-05 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234966+0000", 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1079999999999999e-06 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.204917+0000", 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96995188600000004 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.206597+0000", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0016797380000000001 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.206703+0000", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010574899999999999 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.208806+0000", 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002103443 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213022+0000", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004215791 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213055+0000", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3095000000000001e-05 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213110+0000", 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4880000000000003e-05 2024-09-13T15:02:05.211 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:91482 12.c 12:3b5ee389:::benchmark_data_smithi042_260592_object91481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 244.66343189400001, 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97600144499999997, 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.212 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 91482 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241556+0000", 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241658+0000", 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010255999999999999 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241671+0000", 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2716e-05 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241678+0000", 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7839999999999998e-06 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241721+0000", 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3451999999999997e-05 2024-09-13T15:02:05.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241728+0000", 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2069999999999998e-06 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.208867+0000", 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96713846599999997 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.209010+0000", 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014311 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2", 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.209161+0000", 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015071400000000001 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.210792+0000", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001631071 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217423+0000", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066311879999999997 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217470+0000", 2024-09-13T15:02:05.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7095999999999998e-05 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217558+0000", 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8199e-05 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151137 12.11 12:8d0cd5bb:::benchmark_data_smithi042_260592_object34246:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.219 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.398023435, 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0337329420000001, 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151137 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506964+0000", 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506970+0000", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.708e-06 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506985+0000", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4762e-05 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506992+0000", 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4200000000000001e-06 2024-09-13T15:02:05.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.507029+0000", 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6971999999999998e-05 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.507037+0000", 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4170000000000003e-06 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.514264+0000", 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0072268550000001 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for missing object", 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.514337+0000", 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2965000000000004e-05 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.540613+0000", 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026276239999999999 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.540698+0000", 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5276000000000001e-05 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151161 12.11 12:8e83c0af:::benchmark_data_smithi042_260592_object34270:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.370008528, 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0002382169999999, 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.225 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151161 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534978+0000", 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534986+0000", 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9549999999999992e-06 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.535000+0000", 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.342e-05 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.535007+0000", 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.99e-06 2024-09-13T15:02:05.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.539137+0000", 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041303870000000001 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.539150+0000", 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3162e-05 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522692+0000", 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98354176100000001 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522801+0000", 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010924799999999999 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522871+0000", 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0296000000000006e-05 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530177+0000", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073050820000000001 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.534953+0000", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047767649999999997 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535046+0000", 2024-09-13T15:02:05.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2238999999999996e-05 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535096+0000", 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0439000000000003e-05 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535218+0000", 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012231000000000001 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151191 12.11 12:8e8f485d:::benchmark_data_smithi042_260592_object34300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.332217469, 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.962574547, 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151191 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.233 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572769+0000", 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572777+0000", 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7249999999999997e-06 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572793+0000", 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5471000000000001e-05 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572802+0000", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1269999999999992e-06 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572864+0000", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2045999999999995e-05 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572877+0000", 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3001e-05 2024-09-13T15:02:05.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524707+0000", 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.95182974600000003 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524833+0000", 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012646399999999999 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524963+0000", 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012976099999999999 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530184+0000", 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00522158 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.534996+0000", 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048119549999999997 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535254+0000", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000257781 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535283+0000", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8612e-05 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535346+0000", 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2989000000000003e-05 2024-09-13T15:02:05.239 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151243 12.11 12:8d161ebf:::benchmark_data_smithi042_260592_object34352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.26710004099999, 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90548188600000001, 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.240 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151243 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637887+0000", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637894+0000", 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5079999999999999e-06 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637909+0000", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4885000000000001e-05 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637916+0000", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4729999999999997e-06 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637976+0000", 2024-09-13T15:02:05.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9852999999999998e-05 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637998+0000", 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2798000000000001e-05 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526332+0000", 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.88833333299999995 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526397+0000", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5267999999999999e-05 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526459+0000", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1657999999999998e-05 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531330+0000", 2024-09-13T15:02:05.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048717489999999999 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535397+0000", 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0040661889999999996 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543256+0000", 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0078591389999999994 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543286+0000", 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0521000000000001e-05 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543370+0000", 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4185999999999997e-05 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151343 12.11 12:89bd0aa5:::benchmark_data_smithi042_260592_object34452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.247 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.17467641600001, 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.81331927999999998, 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151343 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730311+0000", 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730316+0000", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1849999999999997e-06 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730325+0000", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5969999999999998e-06 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730330+0000", 2024-09-13T15:02:05.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8579999999999999e-06 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730373+0000", 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3353999999999999e-05 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730381+0000", 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1339999999999998e-06 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527404+0000", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.79702329100000002 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527524+0000", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011975400000000001 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527642+0000", 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011853000000000001 2024-09-13T15:02:05.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531335+0000", 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036927790000000002 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543434+0000", 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012098807 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543488+0000", 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4348000000000002e-05 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543542+0000", 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3609999999999997e-05 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543631+0000", 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9808999999999993e-05 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.254 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151387 12.11 12:8a4c456d:::benchmark_data_smithi042_260592_object34496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.13361607600001, 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80243400399999998, 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151387 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.255 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771371+0000", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771377+0000", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.054e-06 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771390+0000", 2024-09-13T15:02:05.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3179999999999999e-05 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771397+0000", 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4889999999999998e-06 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771460+0000", 2024-09-13T15:02:05.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2290000000000002e-05 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771472+0000", 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2118e-05 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.528824+0000", 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.757352001 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.528932+0000", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107765 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.529042+0000", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110396 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533673+0000", 2024-09-13T15:02:05.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046305799999999996 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.545843+0000", 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012170613 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573759+0000", 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027915598 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573784+0000", 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4757999999999999e-05 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573807+0000", 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2994999999999999e-05 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151403 12.11 12:8c51fcd9:::benchmark_data_smithi042_260592_object34512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.262 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.113844441, 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.78275480399999997, 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151403 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791142+0000", 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791148+0000", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.879e-06 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791160+0000", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1532000000000001e-05 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791166+0000", 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2280000000000003e-06 2024-09-13T15:02:05.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791223+0000", 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7077000000000003e-05 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791236+0000", 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2486999999999999e-05 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530271+0000", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73903509499999998 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530337+0000", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6719999999999998e-05 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530400+0000", 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2718000000000005e-05 2024-09-13T15:02:05.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535001+0000", 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046011639999999996 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560210+0000", 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025208284000000001 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573850+0000", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013640289 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573873+0000", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3402000000000001e-05 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573899+0000", 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5658000000000001e-05 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.269 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151414 12.11 12:8a0a4816:::benchmark_data_smithi042_260592_object34523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.10048960500001, 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76962022699999999, 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.270 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151414 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804497+0000", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804504+0000", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.251e-06 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804522+0000", 2024-09-13T15:02:05.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8397999999999999e-05 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804529+0000", 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.64e-06 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804587+0000", 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8477999999999998e-05 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804598+0000", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0377000000000001e-05 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531367+0000", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72676906399999996 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531474+0000", 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107858 2024-09-13T15:02:05.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531596+0000", 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012136199999999999 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535006+0000", 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034101660000000001 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560242+0000", 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025235807999999998 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574034+0000", 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013791954 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574081+0000", 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7638e-05 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574119+0000", 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7851000000000001e-05 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151428 12.11 12:8a8cc4c2:::benchmark_data_smithi042_260592_object34537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.08704180199999, 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.75627832299999997, 2024-09-13T15:02:05.277 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151428 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817945+0000", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817951+0000", 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.507e-06 2024-09-13T15:02:05.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817962+0000", 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1382e-05 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817969+0000", 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3910000000000003e-06 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.818009+0000", 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0617000000000001e-05 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.818016+0000", 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5320000000000003e-06 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532552+0000", 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.71453657800000003 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532621+0000", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8497000000000005e-05 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532681+0000", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.037e-05 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535353+0000", 2024-09-13T15:02:05.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026720440000000002 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560772+0000", 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025418802000000001 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574179+0000", 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013407211 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574204+0000", 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4708000000000001e-05 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574225+0000", 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1021000000000001e-05 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151478 12.11 12:898a8b15:::benchmark_data_smithi042_260592_object34587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.284 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.03689426299999, 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.70620802299999996, 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151478 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868093+0000", 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868099+0000", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6910000000000002e-06 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868110+0000", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1457999999999999e-05 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868115+0000", 2024-09-13T15:02:05.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1399999999999999e-06 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868161+0000", 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5290000000000002e-05 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868169+0000", 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6200000000000005e-06 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533705+0000", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.665535722 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533811+0000", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106401 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533921+0000", 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110104 2024-09-13T15:02:05.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535483+0000", 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001562 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.569970+0000", 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034486785999999998 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574260+0000", 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042902349999999999 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574283+0000", 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2860000000000001e-05 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574302+0000", 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9160999999999998e-05 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.291 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151495 12.11 12:88ac8106:::benchmark_data_smithi042_260592_object34604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.01691273200001, 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.68633223499999996, 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.292 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151495 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888074+0000", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888082+0000", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1349999999999992e-06 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888098+0000", 2024-09-13T15:02:05.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6269e-05 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888107+0000", 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7010000000000002e-06 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888156+0000", 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8744e-05 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888162+0000", 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8990000000000004e-06 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535524+0000", 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64736161699999994 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535635+0000", 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111013 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535747+0000", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111406 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542042+0000", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0062950209999999996 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570008+0000", 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027966907999999999 2024-09-13T15:02:05.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574369+0000", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043603870000000003 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574390+0000", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1605999999999999e-05 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574408+0000", 2024-09-13T15:02:05.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.768e-05 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151496 12.11 12:8e7a8881:::benchmark_data_smithi042_260592_object34605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 188.013848991, 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.68339865700000002, 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.299 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151496 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:02:05.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891139+0000", 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891144+0000", 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0329999999999999e-06 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891155+0000", 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1548e-05 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891161+0000", 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4709999999999998e-06 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891211+0000", 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9704999999999999e-05 2024-09-13T15:02:05.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891222+0000", 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1589e-05 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.536803+0000", 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64558080200000001 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.536906+0000", 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103293 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.537017+0000", 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011120600000000001 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542046+0000", 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0050282490000000003 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570053+0000", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028007695999999999 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574492+0000", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044390619999999997 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574522+0000", 2024-09-13T15:02:05.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9238999999999999e-05 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574538+0000", 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6653000000000001e-05 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151565 12.11 12:8fde0dfc:::benchmark_data_smithi042_260592_object34674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.306 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 187.931496399, 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.60117407599999995, 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151565 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.307 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973491+0000", 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973497+0000", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2369999999999997e-06 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973507+0000", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0118999999999999e-05 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973530+0000", 2024-09-13T15:02:05.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2552e-05 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973584+0000", 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4262000000000003e-05 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973594+0000", 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.628e-06 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538007+0000", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.564413312 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538114+0000", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010672599999999999 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538225+0000", 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011139800000000001 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542049+0000", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038240710000000001 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570096+0000", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028046482000000001 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574609+0000", 2024-09-13T15:02:05.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045130530000000004 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574635+0000", 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5803999999999999e-05 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574666+0000", 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1739999999999998e-05 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151571 12.11 12:8fdec748:::benchmark_data_smithi042_260592_object34680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 187.924105763, 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.59386337899999997, 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151571 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:05.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980882+0000", 2024-09-13T15:02:05.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980887+0000", 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4550000000000003e-06 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980898+0000", 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1250000000000001e-05 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:05.316 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980904+0000", 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4450000000000004e-06 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980946+0000", 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2249999999999997e-05 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980953+0000", 2024-09-13T15:02:05.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.294e-06 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539191+0000", 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.55823776700000005 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539298+0000", 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106983 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539406+0000", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107598 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542053+0000", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026469359999999999 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570127+0000", 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028074492 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574701+0000", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045736379999999997 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574728+0000", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7042999999999998e-05 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574746+0000", 2024-09-13T15:02:05.320 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8464000000000001e-05 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:05.321 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:02:05.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:05.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:05.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.483 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:05.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:05.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:05.615 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:05.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:05.717 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:05.754 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:05 smithi042 ceph-mon[113926]: pgmap v1290: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T15:02:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:05 smithi042 ceph-mon[108570]: pgmap v1290: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T15:02:05.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:05.895 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:05.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:05.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 97633 97617 98.3913 57.875 0.0139208 0.0101589 2024-09-13T15:02:05.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:05 smithi159 ceph-mon[102519]: pgmap v1290: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-09-13T15:02:06.033 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:06.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:06.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:06.561 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:06.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:06.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:06.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:06.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:06.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:06.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:06.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:06.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:06.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:06.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:06.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:06.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:06.987 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 98452 98436 97.642 51.1875 0.0159411 0.0102369 2024-09-13T15:02:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:07.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:07.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:07.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:07.228 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:07.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:07.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:07.482 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:07.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:02:07.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:07.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:07.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:07.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:07.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:07.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:07 smithi042 ceph-mon[108570]: pgmap v1291: 99 pgs: 99 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:02:07.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:07 smithi042 ceph-mon[113926]: pgmap v1291: 99 pgs: 99 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:02:07.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:07.944 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:07 smithi159 ceph-mon[102519]: pgmap v1291: 99 pgs: 99 active+clean; 5.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:02:07.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:07.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:07.990 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 99442 99426 97.0784 61.875 0.0158494 0.0102948 2024-09-13T15:02:08.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:08.108 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:08.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:08.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:08.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:02:08.259 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:08.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:08.361 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:08.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:08.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:08.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:08.563 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:08.603 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T15:02:08.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:08.687 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:08.789 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/869781984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:08 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/869781984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:08 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/869781984' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.889 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:08 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:02:08.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:08.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:08.982 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:08.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:08.990 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 100416 100400 96.5213 60.875 0.0171554 0.0103553 2024-09-13T15:02:09.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:09.123 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:09.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:09.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:09.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.259 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:09.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:09.360 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:09.396 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:09.499 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:09.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:09.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:09.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:09.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:09.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:09.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:09.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:09.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:09.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:09.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:09.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:09.961 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:02:09.961 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[108570]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T15:02:09.961 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[108570]: pgmap v1293: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T15:02:09.962 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:09] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-09-13T15:02:09.962 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:02:09.962 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[113926]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T15:02:09.962 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:09 smithi042 ceph-mon[113926]: pgmap v1293: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T15:02:09.990 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 101419 101403 96.0087 62.6875 0.00844668 0.0104109 2024-09-13T15:02:10.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:02:10.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:09 smithi159 ceph-mon[102519]: osdmap e361: 8 total, 8 up, 8 in 2024-09-13T15:02:10.002 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:09 smithi159 ceph-mon[102519]: pgmap v1293: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2024-09-13T15:02:10.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:10.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:10.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:10.231 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:10.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:10.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:10.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:10.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:10.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:10.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:10.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:10.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:10.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:10.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:10.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:10.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:10.893 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:10.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:10.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:10.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:10.990 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 102460 102444 95.5467 65.0625 0.0244913 0.010461 2024-09-13T15:02:11.030 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:11.059 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:11.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:11.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.293 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:11.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:11.435 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:11.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:11.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:11.621 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:11.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:11.723 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:11.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:11 smithi159 ceph-mon[102519]: pgmap v1294: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2024-09-13T15:02:11.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:11.836 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:02:11.863 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:11.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:11.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:11.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:11.956 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:11 smithi042 ceph-mon[113926]: pgmap v1294: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2024-09-13T15:02:11.956 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:11 smithi042 ceph-mon[108570]: pgmap v1294: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 44 MiB/s wr, 698 op/s 2024-09-13T15:02:11.990 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 103482 103466 95.0809 63.875 0.00724535 0.0105127 2024-09-13T15:02:12.035 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:12.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.080 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:12.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:12.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:12.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:12.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:12.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:12.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.502 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:02:12.503 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:12.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:12.534 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:02:12.534 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:12.541 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:12.542 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:12.565 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:02:12.565 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:12.594 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:02:12.594 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:12.625 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:02:12.625 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:12.626 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:12.662 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:02:12.662 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:12.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:12.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:12.697 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:02:12.698 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:12.726 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:02:12.727 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:12.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:12.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:12.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:12 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/247742555' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:12.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:12 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/247742555' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:12.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:12.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:12.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:12.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:12.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 104473 104457 94.6 61.9375 0.011208 0.0105656 2024-09-13T15:02:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:12 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/247742555' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:13.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:13.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:13.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:13.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:13.373 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:13.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:13.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:13.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:13.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:13.612 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:13.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:02:13.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:13.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:13.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:13.789 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:13.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:13.891 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:13.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:13.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:13.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 105460 105444 94.1298 61.6875 0.0116033 0.0106173 2024-09-13T15:02:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:13 smithi159 ceph-mon[102519]: pgmap v1295: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:14.054 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:14.062 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:14.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:13 smithi042 ceph-mon[113926]: pgmap v1295: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:14.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:13 smithi042 ceph-mon[108570]: pgmap v1295: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:14.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:14.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:14.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:14.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:14.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:14.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:14.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:14.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:14.580 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:14.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:14.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:14.817 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:14.848 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:14.950 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:14.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:14.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:14.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 106395 106379 93.6271 58.4375 0.00580408 0.0106758 2024-09-13T15:02:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:14.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:15.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:15.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:15.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:15.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:15.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:15.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:15.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:15.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:15.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:15.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:15.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:15.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:15.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:15.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:15.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:15.669 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:15.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:15.834 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:15.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:15.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:15.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:15.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:15.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 16 107365 107349 93.1685 60.625 0.0116017 0.0107285 2024-09-13T15:02:16.014 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:16.112 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:02:16.116 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.183 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.245 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:16.350 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:02:16.350 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:02:16.350 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:02:16.350 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:02:16.351 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:02:16.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:16.459 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:16 smithi159 ceph-mon[102519]: pgmap v1296: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:16.460 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.460 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:16 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.515 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[113926]: pgmap v1296: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[108570]: pgmap v1296: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:02:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:16 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:16.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:16.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:16.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:16.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:16.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:16.916 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:16.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:16.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:16.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:16.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 108331 108315 92.7192 60.375 0.0104756 0.0107804 2024-09-13T15:02:17.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:17.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:17.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:17.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:02:17.203 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:17.244 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:17.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:17.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:17.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:17.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:17.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:17.623 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:17.665 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:17.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:17.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:17.805 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:17.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:17.907 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:17.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:17.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:17.991 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 109295 109279 92.2804 60.25 0.0064654 0.010831 2024-09-13T15:02:18.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.065 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:18.167 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:18.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:18.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:18.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:18.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.453 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:18 smithi159 ceph-mon[102519]: pgmap v1297: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:18.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:18.464 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:18.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:18.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:18 smithi042 ceph-mon[113926]: pgmap v1297: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:18.556 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:18 smithi042 ceph-mon[108570]: pgmap v1297: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:18.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:18.658 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:18.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:18.760 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:18.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:18.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:18.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:18.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:18.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:18.992 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 110225 110209 91.825 58.125 0.0176477 0.010885 2024-09-13T15:02:19.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:19.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:19.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:19.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:19.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:19.357 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:19.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:19.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:19.590 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:19.620 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:19.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:19.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:19.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:19.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:19.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:19.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:19.954 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:19] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.51.0" 2024-09-13T15:02:19.969 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:19.992 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 110800 110784 91.0896 35.9375 0.0264356 0.0109726 2024-09-13T15:02:20.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:20.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:20.010 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:20.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.194 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:02:20.194 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:02:20.215 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[113926]: pgmap v1298: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-13T15:02:20.216 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.216 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.216 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:20.216 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:20.216 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[108570]: pgmap v1298: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-13T15:02:20.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:20.217 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:20 smithi159 ceph-mon[102519]: pgmap v1298: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-13T15:02:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:20.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:02:20.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:20.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:20.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:20.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:20.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:20.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:20.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:20.822 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:20.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:20.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:20.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:20.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:20.992 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 111723 111707 90.6558 57.6875 0.00912051 0.0110263 2024-09-13T15:02:21.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:21.098 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:21.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:21.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:21.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:21 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:21 smithi159 ceph-mon[102519]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T15:02:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:21.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:21.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:21.275 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[113926]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/388396989' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/123561576' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[108570]: osdmap e362: 8 total, 8 up, 8 in 2024-09-13T15:02:21.276 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:21.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:21.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:21.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:21.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:21.527 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:21.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:21.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:21.661 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:21.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:21.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:21.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:21.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:21.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:21.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:21.992 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 112683 112667 90.2628 60 0.0149358 0.0110738 2024-09-13T15:02:22.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:22.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:02:22.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:22.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:22.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:22.201 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:22.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:22.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:22.342 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:22 smithi042 ceph-mon[108570]: pgmap v1300: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:22 smithi042 ceph-mon[113926]: pgmap v1300: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:22.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:22.467 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:22 smithi159 ceph-mon[102519]: pgmap v1300: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s 2024-09-13T15:02:22.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:22.594 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:22.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:22.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:22.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:22.837 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:22.878 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:22.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:22.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:22.993 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 113556 113540 89.8095 54.5625 0.0142383 0.0111298 2024-09-13T15:02:23.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:23.119 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:23.156 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:23.258 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:23.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:23.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:23.398 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:23.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:23.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:23.577 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:23.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:23.679 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:23.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:23.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:23.958 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:23.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:23.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:23.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:23.993 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:02:23.992964+0000 min lat: 0.000998522 max lat: 1.01725 avg lat: 0.011177 2024-09-13T15:02:23.993 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:02:23.993 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 114505 114489 89.4283 59.3125 0.0338411 0.011177 2024-09-13T15:02:24.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:24.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:24.232 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:24.264 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:24 smithi042 ceph-mon[113926]: pgmap v1301: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:24.264 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:24 smithi042 ceph-mon[108570]: pgmap v1301: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:24.271 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:24.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:24.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:24.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:24.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:24 smithi159 ceph-mon[102519]: pgmap v1301: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:24.433 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:02:24.433 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:02:24.433 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:02:24.433 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:161108 12.5 12:a15692a0:::benchmark_data_smithi042_260592_object44217:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:59:04.868074+0000", 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 199.55440052, 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21756971, 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.434 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 161108 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868074+0000", 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.435 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868074+0000", 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868072+0000", 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.436 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868079+0000", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5429999999999999e-06 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868090+0000", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1258e-05 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868096+0000", 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1800000000000001e-06 2024-09-13T15:02:24.437 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868149+0000", 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2729999999999998e-05 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868268+0000", 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011863599999999999 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.438 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.868703+0000", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000435516 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.870949+0000", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022456490000000002 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:04.871537+0000", 2024-09-13T15:02:24.439 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000588738 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085579+0000", 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21404150199999999 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085616+0000", 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6961000000000001e-05 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.440 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:59:05.085643+0000", 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7491000000000002e-05 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216144 12.15 12:af47ab97:::benchmark_data_smithi042_260592_object99253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.680105+0000", 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.742368801, 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.27614016200000002, 2024-09-13T15:02:24.441 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216144 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.442 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680105+0000", 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680105+0000", 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680104+0000", 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.443 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680110+0000", 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5780000000000004e-06 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680124+0000", 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3234999999999999e-05 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.444 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680131+0000", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8589999999999999e-06 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680173+0000", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2809999999999998e-05 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680296+0000", 2024-09-13T15:02:24.445 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012295700000000001 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.680801+0000", 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050423600000000003 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.683865+0000", 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030649459999999998 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.446 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.684770+0000", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00090425100000000001 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.956171+0000", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.27140111500000003 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.956211+0000", 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0229999999999999e-05 2024-09-13T15:02:24.447 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.956246+0000", 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4453999999999999e-05 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216152 12.15 12:ae3890b3:::benchmark_data_smithi042_260592_object99261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.696548+0000", 2024-09-13T15:02:24.448 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.72592642500001, 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.260656524, 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216152 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696548+0000", 2024-09-13T15:02:24.449 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696548+0000", 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696547+0000", 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.450 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696551+0000", 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4290000000000001e-06 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696558+0000", 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1459999999999999e-06 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.451 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696563+0000", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6749999999999999e-06 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696604+0000", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1097999999999999e-05 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.696741+0000", 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013611 2024-09-13T15:02:24.452 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.700082+0000", 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033411769999999999 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.708211+0000", 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0081293839999999999 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.453 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.713845+0000", 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056337210000000004 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957133+0000", 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.243287842 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957172+0000", 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9439e-05 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.454 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957204+0000", 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2172000000000003e-05 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216159 12.5 12:a47786de:::benchmark_data_smithi042_260592_object99268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.702413+0000", 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.720061665, 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.25479322300000001, 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.455 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216159 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702413+0000", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702413+0000", 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.456 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702411+0000", 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702417+0000", 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.913e-06 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.457 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702427+0000", 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6399999999999992e-06 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702432+0000", 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0579999999999997e-06 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.702988+0000", 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00055657700000000005 2024-09-13T15:02:24.458 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703094+0000", 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010556300000000001 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703623+0000", 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00052913000000000005 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.459 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.711386+0000", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.007763425 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.713865+0000", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002478577 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957120+0000", 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24325528599999999 2024-09-13T15:02:24.460 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957161+0000", 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1084000000000002e-05 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.957206+0000", 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4353e-05 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.461 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216160 12.15 12:aaa616d6:::benchmark_data_smithi042_260592_object99269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.703070+0000", 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.71940462800001, 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.25739089900000001, 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216160 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.462 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703070+0000", 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703070+0000", 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703069+0000", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703073+0000", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1030000000000004e-06 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703080+0000", 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.277e-06 2024-09-13T15:02:24.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703086+0000", 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8089999999999998e-06 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703117+0000", 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0710000000000002e-05 2024-09-13T15:02:24.465 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703193+0000", 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6419000000000001e-05 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703692+0000", 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00049868099999999995 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.466 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.708218+0000", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045257830000000002 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.713875+0000", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056576090000000001 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960419+0000", 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24654331900000001 2024-09-13T15:02:24.467 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960442+0000", 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3108999999999998e-05 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960460+0000", 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8821000000000002e-05 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.468 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216162 12.15 12:ab68fad9:::benchmark_data_smithi042_260592_object99271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.703110+0000", 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.71936463, 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.25749051000000001, 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216162 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.469 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703110+0000", 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703110+0000", 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.470 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703109+0000", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703112+0000", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4410000000000002e-06 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703119+0000", 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4230000000000002e-06 2024-09-13T15:02:24.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.703124+0000", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3260000000000003e-06 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706016+0000", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028922319999999998 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706127+0000", 2024-09-13T15:02:24.472 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011092 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706441+0000", 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00031354499999999998 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.711395+0000", 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0049540979999999997 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.473 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.713917+0000", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002522261 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960500+0000", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.246582359 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960557+0000", 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7775999999999997e-05 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.960600+0000", 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2752e-05 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216165 12.5 12:a61ba5de:::benchmark_data_smithi042_260592_object99274:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.706227+0000", 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.716247662, 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.25759674700000001, 2024-09-13T15:02:24.475 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216165 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706227+0000", 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.476 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706227+0000", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706225+0000", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706231+0000", 2024-09-13T15:02:24.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.553e-06 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706243+0000", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1778e-05 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.706249+0000", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3430000000000001e-06 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.708212+0000", 2024-09-13T15:02:24.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0019632220000000001 2024-09-13T15:02:24.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.708283+0000", 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0637999999999997e-05 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.708791+0000", 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00050837199999999999 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.711583+0000", 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027914440000000001 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.713899+0000", 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002315999 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963743+0000", 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24984450699999999 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963788+0000", 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4944999999999997e-05 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963823+0000", 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5181000000000001e-05 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.482 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216173 12.5 12:a60613fa:::benchmark_data_smithi042_260592_object99282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.714085+0000", 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.708389275, 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.249896866, 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216173 2024-09-13T15:02:24.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714085+0000", 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714085+0000", 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714084+0000", 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714091+0000", 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3159999999999999e-06 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714102+0000", 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.077e-05 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714107+0000", 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8500000000000002e-06 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714138+0000", 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1275000000000001e-05 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.486 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714259+0000", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121197 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.714842+0000", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00058254900000000002 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.717981+0000", 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0031395960000000001 2024-09-13T15:02:24.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720254+0000", 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0022732020000000002 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963853+0000", 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.243599125 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963952+0000", 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8633000000000002e-05 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963982+0000", 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9658999999999998e-05 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216175 12.15 12:aba32747:::benchmark_data_smithi042_260592_object99284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.716736+0000", 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.705737709, 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24727703700000001, 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216175 2024-09-13T15:02:24.490 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716736+0000", 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716736+0000", 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716735+0000", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716741+0000", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1260000000000003e-06 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716756+0000", 2024-09-13T15:02:24.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4469e-05 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716763+0000", 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0339999999999999e-06 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716803+0000", 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0414000000000003e-05 2024-09-13T15:02:24.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.716933+0000", 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000129524 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.717463+0000", 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000530387 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.719984+0000", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002521307 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720345+0000", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00036084600000000001 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963890+0000", 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24354503799999999 2024-09-13T15:02:24.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.963978+0000", 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7306999999999996e-05 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.964014+0000", 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5840000000000002e-05 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:216178 12.5 12:a57891aa:::benchmark_data_smithi042_260592_object99287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:01.720793+0000", 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 142.70168160099999, 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.24328304100000001, 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 216178 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.497 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720793+0000", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720793+0000", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720791+0000", 2024-09-13T15:02:24.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720798+0000", 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3790000000000003e-06 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720811+0000", 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3461999999999999e-05 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720822+0000", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.058e-05 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720861+0000", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9530000000000003e-05 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.720975+0000", 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000113886 2024-09-13T15:02:24.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.721434+0000", 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00045860200000000001 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.723805+0000", 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023715429999999998 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.724424+0000", 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00061902400000000003 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.964011+0000", 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.239586929 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.964048+0000", 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7030999999999998e-05 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:01.964076+0000", 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7453999999999998e-05 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110697 13.f 13:f677430e:::benchmark_data_smithi042_286876_object110696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.570759+0000", 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8517153879999997, 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33524619999999999, 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.503 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110697 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570759+0000", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570759+0000", 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570758+0000", 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570827+0000", 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9047999999999993e-05 2024-09-13T15:02:24.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570834+0000", 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8569999999999997e-06 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570854+0000", 2024-09-13T15:02:24.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9777999999999999e-05 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.570891+0000", 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7431000000000001e-05 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571008+0000", 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117008 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:02:24.507 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571116+0000", 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107549 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575608+0000", 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004491732 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.584768+0000", 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0091604519999999995 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.508 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905900+0000", 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.32113141099999998 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905958+0000", 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8714000000000001e-05 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.906005+0000", 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.672e-05 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110702 13.7 13:e1b5f6a6:::benchmark_data_smithi042_286876_object110701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.575238+0000", 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8472361949999998, 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35272493399999999, 2024-09-13T15:02:24.510 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110702 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575238+0000", 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.511 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575238+0000", 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575236+0000", 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:24.512 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575276+0000", 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9697000000000003e-05 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575293+0000", 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7003e-05 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.513 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575300+0000", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2289999999999997e-06 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575627+0000", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000326609 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575751+0000", 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000124399 2024-09-13T15:02:24.514 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575879+0000", 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012787000000000001 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.586346+0000", 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010467495 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.515 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.586387+0000", 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0249e-05 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.927847+0000", 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34146032999999998 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.927927+0000", 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0205999999999997e-05 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.927963+0000", 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5698000000000003e-05 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110703 13.f 13:fffdede6:::benchmark_data_smithi042_286876_object110702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.575307+0000", 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8471676620000004, 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.36009015100000002, 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.517 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110703 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575307+0000", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575307+0000", 2024-09-13T15:02:24.518 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575306+0000", 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575340+0000", 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3739999999999999e-05 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575356+0000", 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6313000000000001e-05 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.575366+0000", 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6670000000000008e-06 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.520 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576296+0000", 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00093009400000000002 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576421+0000", 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000124895 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.580488+0000", 2024-09-13T15:02:24.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0040671040000000002 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588619+0000", 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0081312989999999998 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.592950+0000", 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043312949999999998 2024-09-13T15:02:24.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935317+0000", 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.342366489 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935370+0000", 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2695999999999998e-05 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935397+0000", 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7067999999999999e-05 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110705 13.7 13:ec9f521f:::benchmark_data_smithi042_286876_object110704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.576883+0000", 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8455908049999996, 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35134864599999999, 2024-09-13T15:02:24.524 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110705 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576883+0000", 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576883+0000", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576882+0000", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576985+0000", 2024-09-13T15:02:24.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000102224 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576994+0000", 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2229999999999996e-06 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.576999+0000", 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7250000000000002e-06 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.581212+0000", 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042122690000000003 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.581524+0000", 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00031210899999999998 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.581921+0000", 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039720100000000002 2024-09-13T15:02:24.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588654+0000", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067329470000000004 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588753+0000", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9386e-05 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.928147+0000", 2024-09-13T15:02:24.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.339393642 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.928198+0000", 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0976000000000001e-05 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.928232+0000", 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4019000000000001e-05 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.530 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110707 13.f 13:fd3dafc4:::benchmark_data_smithi042_286876_object110706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.577778+0000", 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8446957399999997, 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35776109, 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110707 2024-09-13T15:02:24.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577778+0000", 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577778+0000", 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577778+0000", 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577831+0000", 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3396999999999997e-05 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577839+0000", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9789999999999993e-06 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.577845+0000", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7069999999999997e-06 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.584790+0000", 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0069453240000000001 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.584969+0000", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000178956 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585134+0000", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000165217 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.594474+0000", 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0093395790000000006 2024-09-13T15:02:24.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600397+0000", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0059234600000000002 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935442+0000", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33504514600000002 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935494+0000", 2024-09-13T15:02:24.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1582000000000002e-05 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935540+0000", 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5503000000000001e-05 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110711 13.f 13:f6ccd480:::benchmark_data_smithi042_286876_object110710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.585439+0000", 2024-09-13T15:02:24.537 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8370348239999998, 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35034428400000001, 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110711 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585439+0000", 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.538 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585439+0000", 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585437+0000", 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:24.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585474+0000", 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6229000000000002e-05 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585491+0000", 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6901999999999998e-05 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585499+0000", 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1810000000000001e-06 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588684+0000", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0031854639999999998 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588871+0000", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000186849 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.589062+0000", 2024-09-13T15:02:24.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000190693 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600406+0000", 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.011344089999999999 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600452+0000", 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5952000000000003e-05 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935719+0000", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33526677100000002 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935754+0000", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5710999999999997e-05 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935784+0000", 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9397e-05 2024-09-13T15:02:24.544 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110712 13.7 13:e929be80:::benchmark_data_smithi042_286876_object110711:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.585506+0000", 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8369683539999997, 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35014874699999998, 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110712 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585506+0000", 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585506+0000", 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585505+0000", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585539+0000", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4204e-05 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585547+0000", 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9759999999999995e-06 2024-09-13T15:02:24.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585553+0000", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.553e-06 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.588903+0000", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033496519999999998 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.589418+0000", 2024-09-13T15:02:24.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00051513999999999998 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.589790+0000", 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00037211500000000001 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600397+0000", 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010607263 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.604830+0000", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004433177 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935580+0000", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33074991799999998 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935622+0000", 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1671000000000001e-05 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935655+0000", 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2704999999999999e-05 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110715 13.7 13:ec5527ef:::benchmark_data_smithi042_286876_object110714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.585559+0000", 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8369153560000004, 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.35020224999999999, 2024-09-13T15:02:24.551 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110715 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585559+0000", 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585559+0000", 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585558+0000", 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585586+0000", 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8045e-05 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585594+0000", 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3359999999999997e-06 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.585600+0000", 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0079999999999999e-06 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.594566+0000", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0089667770000000004 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.594731+0000", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016500900000000001 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.594895+0000", 2024-09-13T15:02:24.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016320199999999999 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600436+0000", 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0055412980000000001 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.604838+0000", 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044019410000000004 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935688+0000", 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33085004000000001 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935730+0000", 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2018000000000003e-05 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935761+0000", 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1090999999999998e-05 2024-09-13T15:02:24.557 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110716 13.7 13:e4c8c0af:::benchmark_data_smithi042_286876_object110715:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.590485+0000", 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8319892859999998, 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34540496999999998, 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.558 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110716 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590485+0000", 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590485+0000", 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590484+0000", 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590526+0000", 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2165e-05 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590544+0000", 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8215000000000001e-05 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.590552+0000", 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1030000000000002e-06 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600461+0000", 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0099085249999999996 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.561 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600666+0000", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000205541 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.600888+0000", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000221886 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.604916+0000", 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0040276899999999996 2024-09-13T15:02:24.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.605089+0000", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000173146 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935791+0000", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33070209 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935854+0000", 2024-09-13T15:02:24.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3208999999999997e-05 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935890+0000", 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5581999999999999e-05 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:110724 13.f 13:f34b66ad:::benchmark_data_smithi042_286876_object110723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.601820+0000", 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 4.8206539030000002, 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33409370500000002, 2024-09-13T15:02:24.564 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 110724 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601820+0000", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:02:24.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601820+0000", 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601819+0000", 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601879+0000", 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9814e-05 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601892+0000", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3468999999999999e-05 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.601898+0000", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1680000000000001e-06 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.604804+0000", 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002905679 2024-09-13T15:02:24.567 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.604963+0000", 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015913200000000001 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.605095+0000", 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013169399999999999 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:02:24.568 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.608723+0000", 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003627856 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.608949+0000", 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00022627699999999999 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935814+0000", 2024-09-13T15:02:24.569 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.326865079 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935883+0000", 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8968999999999996e-05 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.935914+0000", 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1111000000000001e-05 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.570 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.571 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:02:24.571 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:02:24.571 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:02:24.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:24.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:24.691 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:02:24.692 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:24.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:24.727 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:02:24.727 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:24.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:02:24.743 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:24.768 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:02:24.769 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:24.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:02:24.817 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:24.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:24.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:02:24.859 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:24.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:24.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:24.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:24.897 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:02:24.897 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:24.931 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:02:24.931 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:24.969 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:02:24.970 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:24.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:24.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 115404 115388 89.0137 56.1875 0.0264412 0.0112288 2024-09-13T15:02:25.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:25.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:25.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:25.171 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:25.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:25.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:25.313 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:25.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:25.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:25.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1091867327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:25.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:25.568 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:02:25.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1091867327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:25.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1091867327' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:25.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:25.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:25.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:25.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:25.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:25.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:25.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:25.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:25.997 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 116313 116297 88.621 56.8125 0.0163841 0.0112786 2024-09-13T15:02:26.010 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:26.030 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:26.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:26.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:26.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:26.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:26.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:26.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:26.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:26.494 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:26.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:26 smithi159 ceph-mon[102519]: pgmap v1302: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:26.596 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:26.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:26 smithi042 ceph-mon[108570]: pgmap v1302: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:26.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:26 smithi042 ceph-mon[113926]: pgmap v1302: 99 pgs: 99 active+clean; 6.8 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s 2024-09-13T15:02:26.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.636 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:26.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:26.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:26.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:26.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:26.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:26.910 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:26.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:26.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:26.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 117176 117160 88.2022 53.9375 0.0122681 0.0113323 2024-09-13T15:02:27.012 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:27.058 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:27.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:27.160 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:27.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:27.298 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:27.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:27.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:27.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.532 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:27.596 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:27.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:27.758 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:27.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:27.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:27.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:27.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:27.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 117980 117964 87.7504 50.25 0.0100048 0.011391 2024-09-13T15:02:28.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:28.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:28.267 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:28 smithi042 ceph-mon[113926]: pgmap v1303: 99 pgs: 99 active+clean; 7.0 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T15:02:28.267 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:28 smithi042 ceph-mon[108570]: pgmap v1303: 99 pgs: 99 active+clean; 7.0 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T15:02:28.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.322 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:28.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:28.424 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:28 smithi159 ceph-mon[102519]: pgmap v1303: 99 pgs: 99 active+clean; 7.0 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-09-13T15:02:28.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:28.584 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:28.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:28.686 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:28.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:28.826 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:28.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:28.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:28.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:28.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:28.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:28.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 118907 118891 87.3997 57.9375 0.00876566 0.0114362 2024-09-13T15:02:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.008 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:29.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:29.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:29.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:29.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:29.215 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:02:29.251 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:29.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:29.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:29.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:29.428 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:29.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:29.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:29.429 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.430 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:29.532 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:29.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:29.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:02:29.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:02:29.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:29.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:29.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:29.767 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:29.767 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:29.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:29.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:29.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:29.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:29.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 119820 119804 87.0469 57.0625 0.0147983 0.011483 2024-09-13T15:02:30.000 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:30.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:29] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T15:02:30.102 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.141 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:30.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:02:30.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:30.274 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:30.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:30.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:30 smithi159 ceph-mon[102519]: pgmap v1304: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:02:30.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:30 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:02:30.475 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:30 smithi159 ceph-mon[102519]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T15:02:30.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:02:30.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:30.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[113926]: pgmap v1304: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:02:30.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:02:30.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[113926]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T15:02:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[108570]: pgmap v1304: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s 2024-09-13T15:02:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2905429854' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:02:30.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:30 smithi042 ceph-mon[108570]: osdmap e363: 8 total, 8 up, 8 in 2024-09-13T15:02:30.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:30.685 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:30.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.823 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:30.887 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:30.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:30.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:30.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:30.998 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 16 120720 120704 86.6929 56.25 0.00856015 0.0115287 2024-09-13T15:02:31.085 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:31.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:31.230 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:31.273 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:31.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:31.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:31.477 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:31.480 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:31.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:31.593 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:31.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:31.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:31.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:31.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:31.835 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:31.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:31.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:31.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:31.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:31.999 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 121645 121629 86.3642 57.8125 0.0403184 0.0115733 2024-09-13T15:02:32.013 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:32.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:32.115 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:32.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:32.243 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:02:32.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[113926]: pgmap v1306: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1923866055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[108570]: pgmap v1306: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1923866055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:32 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:32.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:32.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:32.562 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:32.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:32.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:32.667 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:32 smithi159 ceph-mon[102519]: pgmap v1306: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2024-09-13T15:02:32.667 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1923866055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.667 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:32 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:02:32.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:32.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:32.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:32.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:32.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:32.945 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:32.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:32.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:32.999 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 122576 122560 86.0476 58.1875 0.0137017 0.0116161 2024-09-13T15:02:33.084 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:33.123 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:33.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:33.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:33.333 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:33.333 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:33.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:33.403 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:33.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:33.505 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:33.547 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:33.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:33.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:33.566 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[113926]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T15:02:33.566 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:33.567 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:33.567 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:33 smithi042 ceph-mon[108570]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T15:02:33.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:33.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:33 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:02:33.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:02:33.655 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:33 smithi159 ceph-mon[102519]: osdmap e364: 8 total, 8 up, 8 in 2024-09-13T15:02:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:33.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:33.788 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:33.822 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:33.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:33.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:33.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:33.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:33.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:33.999 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0162 2024-09-13T15:02:33.999 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 123498 2024-09-13T15:02:33.999 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 85.747 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 40.1149 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 233.438 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 12.375 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1371 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 641.838 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3735 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 198 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0116576 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0175151 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 1.01725 2024-09-13T15:02:34.000 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.000998522 2024-09-13T15:02:34.004 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:02:34.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:02:34.068 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:34.104 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:34.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:34.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:34 smithi042 ceph-mon[108570]: pgmap v1308: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:02:34.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:34 smithi042 ceph-mon[113926]: pgmap v1308: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:02:34.360 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:02:34.371 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:34.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:34.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:34.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:34.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:34.636 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:34 smithi159 ceph-mon[102519]: pgmap v1308: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-13T15:02:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:34.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:34.746 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:34.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:34.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:34.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:34.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:34.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:35.021 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:35.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:35.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:35.161 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:35.197 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:35.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:35.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:35.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:35.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:35.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:35.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:35.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:35.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:35.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:35.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:35.906 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:35.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:35.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:36.008 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:36.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:36.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:36.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:36.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:36.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:36.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:36.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:36.421 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:02:36.421 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:02:36.422 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:36.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:36.553 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:36.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:36.594 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:36 smithi042 ceph-mon[113926]: pgmap v1309: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2024-09-13T15:02:36.594 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.594 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:36 smithi042 ceph-mon[108570]: pgmap v1309: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2024-09-13T15:02:36.594 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:36.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:36 smithi159 ceph-mon[102519]: pgmap v1309: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 43 MiB/s wr, 693 op/s 2024-09-13T15:02:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:36.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:36.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:36.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:36.996 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:36.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:36.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:37.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:37.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:02:37.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.299 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:02:37.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.396 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1602931003' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:37 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.559 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1602931003' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.559 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:37 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.561 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1602931003' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.562 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:37 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:02:37.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:37.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:37.767 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:37.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:37.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:37.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:37.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:38.037 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:38.074 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:38.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:38.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:38.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:38.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:02:38.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:38.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:38.502 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:38.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[108570]: pgmap v1310: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 51 MiB/s wr, 988 op/s 2024-09-13T15:02:38.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:38.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[108570]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T15:02:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[113926]: pgmap v1310: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 51 MiB/s wr, 988 op/s 2024-09-13T15:02:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:38 smithi042 ceph-mon[113926]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T15:02:38.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:38.636 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:38.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:38.740 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:38 smithi159 ceph-mon[102519]: pgmap v1310: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 51 MiB/s wr, 988 op/s 2024-09-13T15:02:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:38 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:02:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:38 smithi159 ceph-mon[102519]: osdmap e365: 8 total, 8 up, 8 in 2024-09-13T15:02:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:38.874 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:38.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:38.945 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:38.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:38.954 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:02:38.955 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:38.969 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:39.000 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:02:39.000 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:39.042 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:02:39.042 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:39.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:39.076 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:02:39.076 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:39.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:39.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:39.112 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:02:39.112 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:39.149 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:02:39.149 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:39.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:39.181 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:02:39.182 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:39.214 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:02:39.214 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.225 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:39.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:39.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:02:39.327 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:39.412 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:39.514 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:39.542 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:39.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:39.562 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1346036929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:39.562 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1346036929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:39.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:39.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:39.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1346036929' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:39.790 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:39.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:39.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:39] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T15:02:39.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:39.979 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:39.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:39.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:40.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:40.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:40.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:40.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:40.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:40.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:40.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:40.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:40.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:40.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:40 smithi042 ceph-mon[113926]: pgmap v1312: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 1.06k op/s 2024-09-13T15:02:40.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:40 smithi042 ceph-mon[108570]: pgmap v1312: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 1.06k op/s 2024-09-13T15:02:40.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:40.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:40.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:40 smithi159 ceph-mon[102519]: pgmap v1312: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 127 B/s rd, 50 MiB/s wr, 1.06k op/s 2024-09-13T15:02:40.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:40.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:40.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:40.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:40.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:40.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:41.055 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:41.098 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:41.200 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:41.239 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:41.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:41.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:41.506 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:41.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:41.608 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.641 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:41.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:41.783 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:41.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:41.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:41.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:42.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:42.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:42.164 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:42.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:42.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:42.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.477 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:42.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:42 smithi042 ceph-mon[113926]: pgmap v1313: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 121 B/s rd, 23 MiB/s wr, 693 op/s 2024-09-13T15:02:42.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:42 smithi042 ceph-mon[108570]: pgmap v1313: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 121 B/s rd, 23 MiB/s wr, 693 op/s 2024-09-13T15:02:42.616 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:42.718 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:42.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:42 smithi159 ceph-mon[102519]: pgmap v1313: 99 pgs: 99 active+clean; 7.4 GiB data, 30 GiB used, 685 GiB / 715 GiB avail; 121 B/s rd, 23 MiB/s wr, 693 op/s 2024-09-13T15:02:42.754 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:42.857 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:42.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:42.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:42.932 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:42.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:43.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:43.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:43.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:43.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:43.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:43.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:43.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:43.417 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:43.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:43.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:43.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:43.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:43.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:43.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:43.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:43.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:43.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:43.954 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:43.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:43.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:44.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:02:44.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:44.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:44.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:44.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.254 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:44.340 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:02:44.340 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:133460 13.9 13:9826ca46:::benchmark_data_smithi042_286876_object9959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:44.305579+0000", 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.026448079999999999, 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.026475556000000001, 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 133460 2024-09-13T15:02:44.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305579+0000", 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305579+0000", 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305581+0000", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1730000000000002e-06 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305588+0000", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3170000000000004e-06 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305608+0000", 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9245e-05 2024-09-13T15:02:44.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305616+0000", 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8859999999999993e-06 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.305810+0000", 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000193477 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.306135+0000", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00032528100000000002 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.306496+0000", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00036099999999999999 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.311198+0000", 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047017359999999998 2024-09-13T15:02:44.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.311249+0000", 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0614e-05 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:133471 13.9 13:9f9d99fe:::benchmark_data_smithi042_286876_object9970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:44.315549+0000", 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.016477616, 2024-09-13T15:02:44.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016586744, 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 133471 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:02:44.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315549+0000", 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315549+0000", 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315550+0000", 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5600000000000001e-07 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315552+0000", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.621e-06 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315559+0000", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5130000000000002e-06 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315564+0000", 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.305e-06 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.315909+0000", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034473400000000002 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.316059+0000", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149829 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.316180+0000", 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120875 2024-09-13T15:02:44.350 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45788.0:133471 13.9 e365/354)", 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:44.331911+0000", 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.000115782, 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00028852899999999999, 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331911+0000", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331911+0000", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331913+0000", 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4139999999999999e-06 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331918+0000", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9159999999999998e-06 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331937+0000", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8658999999999999e-05 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331945+0000", 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.106e-06 2024-09-13T15:02:44.353 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:133481 13.9 13.9e27ffe9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e365)", 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:44.331989+0000", 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 3.8359999999999999e-05, 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025287699999999998, 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:02:44.354 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 133481 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331989+0000", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331989+0000", 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:02:44.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.331990+0000", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.243e-06 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.332003+0000", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3193000000000001e-05 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.332013+0000", 2024-09-13T15:02:44.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0183999999999999e-05 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:44.332020+0000", 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5880000000000003e-06 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 4 2024-09-13T15:02:44.357 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:02:44.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:44.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:44.404 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:44 smithi042 ceph-mon[108570]: pgmap v1314: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:44.405 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:44 smithi042 ceph-mon[113926]: pgmap v1314: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:44.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.522 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:44.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:44.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:44.665 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:44.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:44.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:44 smithi159 ceph-mon[102519]: pgmap v1314: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:44.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:44.794 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:44.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:44.896 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:44.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:44.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:45.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:45.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:45.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:45.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:45.341 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:45.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:45.344 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:02:45.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:45.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:45.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:45 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/492090766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:45 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:45 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/492090766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:45 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:45.749 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:45 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/492090766' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:45 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:02:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:45.851 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:45.888 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:45.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:45.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:46.026 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:46.128 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:46.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:46.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:46.267 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:46.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:02:46.302 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:46.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:46.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:02:46.468 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:46.525 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[108570]: pgmap v1315: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:46.525 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:02:46.525 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[108570]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T15:02:46.526 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[113926]: pgmap v1315: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:46.526 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:02:46.526 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:46 smithi042 ceph-mon[113926]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T15:02:46.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:46.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:46.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:46.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:46.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:46 smithi159 ceph-mon[102519]: pgmap v1315: 99 pgs: 99 active+clean; 7.1 GiB data, 29 GiB used, 686 GiB / 715 GiB avail; 102 B/s rd, 16 MiB/s wr, 956 op/s 2024-09-13T15:02:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:46 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:02:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:46 smithi159 ceph-mon[102519]: osdmap e366: 8 total, 8 up, 8 in 2024-09-13T15:02:46.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:46.880 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:46.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:46.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:46.961 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:46.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:46.993 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:47.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:47.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.171 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:47.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.308 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.410 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:47.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:47.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:47.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:47.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:47.734 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:47.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:47.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:47.869 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:47.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:47.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:47.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:48.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:48.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:48.108 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:48.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:48.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:48.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:48.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:48.426 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:48.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:48.565 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:48.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:48.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:48 smithi042 ceph-mon[113926]: pgmap v1317: 99 pgs: 99 active+clean; 6.9 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 1.2 MiB/s wr, 974 op/s 2024-09-13T15:02:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:48 smithi042 ceph-mon[108570]: pgmap v1317: 99 pgs: 99 active+clean; 6.9 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 1.2 MiB/s wr, 974 op/s 2024-09-13T15:02:48.667 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:48.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:48.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:48 smithi159 ceph-mon[102519]: pgmap v1317: 99 pgs: 99 active+clean; 6.9 GiB data, 29 GiB used, 687 GiB / 715 GiB avail; 1.2 MiB/s wr, 974 op/s 2024-09-13T15:02:48.805 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:48.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:48.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:48.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:48.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:48.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:49.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:49.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:49.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:49.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:49.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:49.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.368 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:49.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:49.381 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:49.381 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:49.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:49.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:02:49.506 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:49.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:49.603 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:02:49.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:49.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:49.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:49.781 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:49.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:49.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:49.912 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:49.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:50.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:50.046 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:49] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T15:02:50.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:50.177 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:50.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:50.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:50.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:50.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:50.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:50.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:50.579 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:50.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:50.713 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:50 smithi159 ceph-mon[102519]: pgmap v1318: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T15:02:50.714 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:50 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:50.714 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:50 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:02:50.714 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:50.714 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:50 smithi159 ceph-mon[102519]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T15:02:50.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:50.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:50.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[113926]: pgmap v1318: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T15:02:50.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[113926]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[108570]: pgmap v1318: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 976 op/s 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3347275767' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:02:50.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:50 smithi042 ceph-mon[108570]: osdmap e367: 8 total, 8 up, 8 in 2024-09-13T15:02:50.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:50.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:50.862 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:50.963 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:51.114 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:51.148 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:51.250 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:51.285 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:51.382 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:51.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:51.425 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:51.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:51.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:51.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.700 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:51.801 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:51.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:51.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:51.985 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:51.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:52.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[113926]: pgmap v1320: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T15:02:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[108570]: pgmap v1320: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T15:02:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:52.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:52.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:52.201 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:02:52.202 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:02:52.202 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:02:52.202 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:02:52.202 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:02:52.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:51 smithi159 ceph-mon[102519]: pgmap v1320: 99 pgs: 99 active+clean; 6.8 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2024-09-13T15:02:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:02:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:02:52.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:52.378 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:52.407 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:02:52.411 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:52.513 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:52.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:52.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:52.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:52.749 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:02:52.749 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:02:52.785 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:02:52.785 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:02:52.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:52.820 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:02:52.820 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:02:52.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:52.848 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:02:52.849 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:02:52.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:02:52.885 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:02:52.922 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:02:52.923 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:02:52.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:52.956 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:02:52.957 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:02:52.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:52.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:52.998 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:02:52.998 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:02:53.009 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3189352774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:53.009 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3189352774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:53.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:53.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:53.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:53.232 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3189352774' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:02:53.276 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:53.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:53.378 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:53.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:02:53.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:53.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:53.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:53.646 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:53.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:53.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:53.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:53.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:53.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:53.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:53.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:53.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:54.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:54.085 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:54.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:53 smithi042 ceph-mon[113926]: pgmap v1321: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:02:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:53 smithi042 ceph-mon[108570]: pgmap v1321: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:02:54.188 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:54.221 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:54.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:53 smithi159 ceph-mon[102519]: pgmap v1321: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-09-13T15:02:54.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.361 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:54.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:54.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:54.495 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:54.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:54.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:54.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:54.740 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:54.772 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:54.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:54.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:54.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:55.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:55.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:55.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:55.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:55.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:55.275 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:55.318 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:55.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:55.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:55.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:55.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:55.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:55.595 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:55.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.737 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:55.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:55.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:55.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:55.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:55.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:55.979 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:56.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:56.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:56.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:55 smithi042 ceph-mon[108570]: pgmap v1322: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T15:02:56.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:55 smithi042 ceph-mon[113926]: pgmap v1322: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T15:02:56.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.148 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:55 smithi159 ceph-mon[102519]: pgmap v1322: 99 pgs: 99 active+clean; 6.5 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 952 op/s 2024-09-13T15:02:56.250 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:56.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:56.383 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:56.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:56.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:56.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:56.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:56.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:56.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:56.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:56.943 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:56.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:56.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:56.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:57.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:57.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.123 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:57.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:57.366 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:57.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.405 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:57.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:57.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:57.639 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:57.685 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:57.700 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:02:57.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:57.847 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:57.898 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:57 smithi042 ceph-mon[108570]: pgmap v1323: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:02:57.898 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:57 smithi042 ceph-mon[113926]: pgmap v1323: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:02:57.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:57.987 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:57.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:57.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:58.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:58.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:58.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:58.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:57 smithi159 ceph-mon[102519]: pgmap v1323: 99 pgs: 99 active+clean; 6.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:02:58.259 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:58.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:58.360 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:58.400 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:58.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:02:58.502 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:02:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.537 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:58.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:02:58.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.687 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:58.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:58.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:02:58.819 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:02:58.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:02:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:58.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:58.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:02:59.067 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:59.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:59.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:59.138 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:02:59.200 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:59.239 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:59.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:59.342 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:02:59.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:59.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:02:59.489 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:02:59.520 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:59.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:02:59.623 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:02:59.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:59.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:02:59.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:02:59.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:02:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:02:59.841 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:02:59.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:02:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:02:59.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:02:59.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:00.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:00.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[113926]: pgmap v1324: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:03:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4232797628' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[108570]: pgmap v1324: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:03:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4232797628' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:02:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:02:59] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.51.0" 2024-09-13T15:03:00.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.226 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:00.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:59 smithi159 ceph-mon[102519]: pgmap v1324: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-13T15:03:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4232797628' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:02:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:03:00.328 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:00.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:00.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:00.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:00.605 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:00.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.641 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:00.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:00.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:00.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:00.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:00.902 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:03:00.902 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:00 smithi042 ceph-mon[108570]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T15:03:00.903 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:03:00.903 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:00 smithi042 ceph-mon[113926]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T15:03:00.921 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:00.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:00.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:01.023 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:01.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:01.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:01.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:03:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:00 smithi159 ceph-mon[102519]: osdmap e368: 8 total, 8 up, 8 in 2024-09-13T15:03:01.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:01.342 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:01.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.343 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:01.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:01.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:01.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.618 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:01.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:01.719 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:01.754 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:01.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:01.856 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:01.893 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:01.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:01.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:01.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:02.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:02.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:02.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:01 smithi159 ceph-mon[102519]: pgmap v1326: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T15:03:02.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:02.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:02.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:01 smithi042 ceph-mon[108570]: pgmap v1326: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T15:03:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:01 smithi042 ceph-mon[113926]: pgmap v1326: 99 pgs: 99 active+clean; 6.2 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-13T15:03:02.387 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:03:02.387 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:02.408 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:02.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:02.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:02.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:02.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.605 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:02.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:02.706 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:02.752 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:02.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:02.891 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:02.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:02.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:02.993 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:03.022 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:03.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:03.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.145 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:03.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:03.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.271 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:03.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:03.286 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:03:03.286 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:03:03.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.395 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:03.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:03.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:03.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:03.638 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:03.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:03.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:03.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:03.813 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:03.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:03.877 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[108570]: pgmap v1327: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T15:03:03.877 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1786010599' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:03.877 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:03.878 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[113926]: pgmap v1327: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T15:03:03.878 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1786010599' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:03.878 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:03.878 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:03:03.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:03.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:03.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:03.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:03:04.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.085 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:04.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:03 smithi159 ceph-mon[102519]: pgmap v1327: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-09-13T15:03:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1786010599' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:03:04.259 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:04.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:04.353 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:04.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:04.493 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:04.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:04.595 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:04.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:04.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.753 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.874 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:04.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:04.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:04.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:04.962 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:04.962 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:04.963 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[108570]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T15:03:04.963 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:04.963 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:04.963 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:04 smithi042 ceph-mon[113926]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T15:03:05.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:05.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:05.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:05.197 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:05.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:04 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:04 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:03:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:04 smithi159 ceph-mon[102519]: osdmap e369: 8 total, 8 up, 8 in 2024-09-13T15:03:05.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:05.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:05.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:05.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:05.462 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:05.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:05.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:05.704 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:05.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:05.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:05.839 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:05.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:05.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:05.971 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:05.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:06.022 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:06.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:06.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:05 smithi159 ceph-mon[102519]: pgmap v1329: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:03:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:06.300 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:06.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[108570]: pgmap v1329: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:03:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[113926]: pgmap v1329: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-09-13T15:03:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:06.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:06.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.440 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:06.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:06.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:06.581 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:06.682 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:06.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:03:06.823 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:06.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:06.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:06.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:07.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:07.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:07.120 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:07.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:07.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:07.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:07.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:07.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:07.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:07.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:07.540 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:07.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.674 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:07.727 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:07.829 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:07.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:07.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:07.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:07.926 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:07 smithi042 ceph-mon[108570]: pgmap v1330: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:03:07.926 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:07 smithi042 ceph-mon[113926]: pgmap v1330: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:03:07.967 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:07.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:08.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:08.101 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:08.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:08.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:08.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:07 smithi159 ceph-mon[102519]: pgmap v1330: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-13T15:03:08.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:08.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:08.417 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:08.519 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:08.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:08.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:08.657 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:08.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:08.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:08.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:08.944 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:08.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:08.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.080 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:09.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:09.255 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:09.356 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.491 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:09.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.628 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:09.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:09.717 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:03:09.717 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:09.753 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:03:09.753 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:09.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:09.786 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:03:09.786 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:09.801 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:09.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:03:09.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:09.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:03:09.860 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:09.897 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:03:09.897 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:09.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:09.935 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:09.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:09.941 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:03:09.941 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:09.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:09 smithi042 ceph-mon[113926]: pgmap v1331: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T15:03:09.970 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/359156171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:09.970 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:09 smithi042 ceph-mon[108570]: pgmap v1331: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T15:03:09.970 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/359156171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:09.971 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:09] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T15:03:09.971 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:03:09.971 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:03:09.971 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:157415 13.c 13:33451e36:::benchmark_data_smithi042_286876_object33914:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:09.957048+0000", 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.01161308, 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011645150999999999, 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:03:09.972 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 157415 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957048+0000", 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957048+0000", 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.973 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957050+0000", 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.699e-06 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957057+0000", 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5649999999999998e-06 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.974 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957074+0000", 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7320999999999999e-05 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957082+0000", 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3690000000000007e-06 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:03:09.975 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957257+0000", 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017468299999999999 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957664+0000", 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00040733000000000002 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957920+0000", 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025581200000000001 2024-09-13T15:03:09.976 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.967617+0000", 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096970749999999994 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.967680+0000", 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2823000000000005e-05 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.977 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:157417 13.4 e369/344 13:27e2ea66:::benchmark_data_smithi042_286876_object33916:head v 369'9662, mlcod=369'9660)", 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:09.957644+0000", 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.01101689, 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011142269999999999, 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.978 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957644+0000", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957644+0000", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957645+0000", 2024-09-13T15:03:09.979 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6599999999999995e-07 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957652+0000", 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4489999999999996e-06 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957675+0000", 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3523000000000002e-05 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.980 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957682+0000", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6769999999999998e-06 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957763+0000", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0749999999999998e-05 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.957782+0000", 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9738999999999999e-05 2024-09-13T15:03:09.981 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:157423 13.4 e369/344 13:2cc05b16:::benchmark_data_smithi042_286876_object33922:head v 369'9663, mlcod=369'9660)", 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:09.962657+0000", 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0060039209999999997, 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061818180000000004, 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:03:09.982 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962657+0000", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962657+0000", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962659+0000", 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.128e-06 2024-09-13T15:03:09.983 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962666+0000", 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1210000000000001e-06 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962679+0000", 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2848e-05 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.984 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.962686+0000", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1289999999999998e-06 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965180+0000", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024943460000000001 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965200+0000", 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0063000000000001e-05 2024-09-13T15:03:09.985 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45788.0:157415 13.c e369/344 ondisk, result = 0)", 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:09.965886+0000", 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.00277582, 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003006356, 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:03:09.986 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965886+0000", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965886+0000", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965887+0000", 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1379999999999999e-06 2024-09-13T15:03:09.987 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965892+0000", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6409999999999999e-06 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965905+0000", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2607e-05 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.965914+0000", 2024-09-13T15:03:09.988 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.738e-06 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.967568+0000", 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016539530000000001 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:09.967594+0000", 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6849999999999999e-05 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:03:09.989 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 4 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:03:09.990 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:10.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:10.076 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:10.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:10.212 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:09 smithi159 ceph-mon[102519]: pgmap v1331: 99 pgs: 99 active+clean; 5.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-09-13T15:03:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/359156171' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:10.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:10.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.352 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:10.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:10.453 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:10.490 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:10.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:10.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.628 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:10.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:10.657 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:03:10.659 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:03:10.729 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:10.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:10.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:10.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:10.901 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:10.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:11.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:11.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:11.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:11.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:11.187 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:11.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:11.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:11.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.459 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:11.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:11.560 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:11.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:11.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:11.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:11.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:11.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:11.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:11.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:11.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:11.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:11.975 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.005 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:12.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:12.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:11 smithi042 ceph-mon[108570]: from='client.45938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:11 smithi042 ceph-mon[108570]: pgmap v1332: 99 pgs: 99 active+clean; 5.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T15:03:12.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:11 smithi042 ceph-mon[113926]: from='client.45938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:11 smithi042 ceph-mon[113926]: pgmap v1332: 99 pgs: 99 active+clean; 5.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T15:03:12.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:03:12.107 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:12.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:12.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:11 smithi159 ceph-mon[102519]: from='client.45938 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:11 smithi159 ceph-mon[102519]: pgmap v1332: 99 pgs: 99 active+clean; 5.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-09-13T15:03:12.289 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:12.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:12.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:12.427 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:12.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:12.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:12.561 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:12.569 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:12.663 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:12.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:12.798 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:12.837 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:12.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:12.982 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:12.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.084 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:13.127 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:13.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:13.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:13.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:13.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.410 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:13.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:13.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:13.546 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:13.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:13.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:13.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:13.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:13.875 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1331,"stamp":"2024-09-13T15:03:08.661027+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6102587389,"num_objects":93136,"num_object_clones":0,"num_object_copies":279408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93136,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153952,"num_write_kb":7903906,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":5182,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127486,"ondisk_log_size":127486,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":26203904,"kb_used_data":18343288,"kb_used_omap":33,"kb_used_meta":7860190,"kb_avail":723757312,"statfs":{"total":767960285184,"available":741127487488,"internally_reserved":0,"allocated":18783526912,"data_stored":18767457634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33996,"internal_metadata":8048835380},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-596377600,"num_objects":-9100,"num_object_clones":0,"num_object_copies":-27300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9100,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.820533"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":564,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879972+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:03:03.879972+0000","last_peered":"2024-09-13T15:03:03.879972+0000","last_clean":"2024-09-13T15:03:03.879972+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:03:03.879972+0000","last_undegraded":"2024-09-13T15:03:03.879972+0000","last_fullsized":"2024-09-13T15:03:03.879972+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":61,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837592+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:02:59.837592+0000","last_peered":"2024-09-13T15:02:59.837592+0000","last_clean":"2024-09-13T15:02:59.837592+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:02:59.837592+0000","last_undegraded":"2024-09-13T15:02:59.837592+0000","last_fullsized":"2024-09-13T15:02:59.837592+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":128,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880256+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:03:03.880256+0000","last_peered":"2024-09-13T15:03:03.880256+0000","last_clean":"2024-09-13T15:03:03.880256+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:03:03.880256+0000","last_undegraded":"2024-09-13T15:03:03.880256+0000","last_fullsized":"2024-09-13T15:03:03.880256+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":150,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882285+0000","last_change":"2024-09-13T15:00:58.195537+0000","last_active":"2024-09-13T15:03:03.882285+0000","last_peered":"2024-09-13T15:03:03.882285+0000","last_clean":"2024-09-13T15:03:03.882285+0000","last_became_active":"2024-09-13T15:00:58.194028+0000","last_became_peered":"2024-09-13T15:00:58.194028+0000","last_unstale":"2024-09-13T15:03:03.882285+0000","last_undegraded":"2024-09-13T15:03:03.882285+0000","last_fullsized":"2024-09-13T15:03:03.882285+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:47.298186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":150,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882939+0000","last_change":"2024-09-13T15:00:58.197669+0000","last_active":"2024-09-13T15:03:03.882939+0000","last_peered":"2024-09-13T15:03:03.882939+0000","last_clean":"2024-09-13T15:03:03.882939+0000","last_became_active":"2024-09-13T15:00:58.196512+0000","last_became_peered":"2024-09-13T15:00:58.196512+0000","last_unstale":"2024-09-13T15:03:03.882939+0000","last_undegraded":"2024-09-13T15:03:03.882939+0000","last_fullsized":"2024-09-13T15:03:03.882939+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:15.533805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837326+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:02:59.837326+0000","last_peered":"2024-09-13T15:02:59.837326+0000","last_clean":"2024-09-13T15:02:59.837326+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:02:59.837326+0000","last_undegraded":"2024-09-13T15:02:59.837326+0000","last_fullsized":"2024-09-13T15:02:59.837326+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":590,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880163+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:03:03.880163+0000","last_peered":"2024-09-13T15:03:03.880163+0000","last_clean":"2024-09-13T15:03:03.880163+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:03:03.880163+0000","last_undegraded":"2024-09-13T15:03:03.880163+0000","last_fullsized":"2024-09-13T15:03:03.880163+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":442,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841578+0000","last_change":"2024-09-13T15:00:58.197147+0000","last_active":"2024-09-13T15:02:59.841578+0000","last_peered":"2024-09-13T15:02:59.841578+0000","last_clean":"2024-09-13T15:02:59.841578+0000","last_became_active":"2024-09-13T15:00:58.196810+0000","last_became_peered":"2024-09-13T15:00:58.196810+0000","last_unstale":"2024-09-13T15:02:59.841578+0000","last_undegraded":"2024-09-13T15:02:59.841578+0000","last_fullsized":"2024-09-13T15:02:59.841578+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:03.062492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"369'9600","reported_seq":9635,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.113780+0000","last_change":"2024-09-13T15:00:51.158623+0000","last_active":"2024-09-13T15:03:06.113780+0000","last_peered":"2024-09-13T15:03:06.113780+0000","last_clean":"2024-09-13T15:03:06.113780+0000","last_became_active":"2024-09-13T15:00:51.158283+0000","last_became_peered":"2024-09-13T15:00:51.158283+0000","last_unstale":"2024-09-13T15:03:06.113780+0000","last_undegraded":"2024-09-13T15:03:06.113780+0000","last_fullsized":"2024-09-13T15:03:06.113780+0000","mapping_epoch":344,"log_start":"352'1400","ondisk_log_start":"352'1400","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8200,"log_dups_size":0,"ondisk_log_size":8200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:37:32.843875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387186688,"num_objects":5908,"num_object_clones":0,"num_object_copies":17724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":70,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.843019+0000","last_change":"2024-09-13T15:00:59.188914+0000","last_active":"2024-09-13T15:02:59.843019+0000","last_peered":"2024-09-13T15:02:59.843019+0000","last_clean":"2024-09-13T15:02:59.843019+0000","last_became_active":"2024-09-13T15:00:59.188606+0000","last_became_peered":"2024-09-13T15:00:59.188606+0000","last_unstale":"2024-09-13T15:02:59.843019+0000","last_undegraded":"2024-09-13T15:02:59.843019+0000","last_fullsized":"2024-09-13T15:02:59.843019+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:32.701420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":26,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841431+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:02:59.841431+0000","last_peered":"2024-09-13T15:02:59.841431+0000","last_clean":"2024-09-13T15:02:59.841431+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:02:59.841431+0000","last_undegraded":"2024-09-13T15:02:59.841431+0000","last_fullsized":"2024-09-13T15:02:59.841431+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.0","version":"369'9505","reported_seq":9567,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:04.852448+0000","last_change":"2024-09-13T15:00:51.638677+0000","last_active":"2024-09-13T15:03:04.852448+0000","last_peered":"2024-09-13T15:03:04.852448+0000","last_clean":"2024-09-13T15:03:04.852448+0000","last_became_active":"2024-09-13T15:00:51.638338+0000","last_became_peered":"2024-09-13T15:00:51.638338+0000","last_unstale":"2024-09-13T15:03:04.852448+0000","last_undegraded":"2024-09-13T15:03:04.852448+0000","last_fullsized":"2024-09-13T15:03:04.852448+0000","mapping_epoch":344,"log_start":"351'1300","ondisk_log_start":"351'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8205,"log_dups_size":0,"ondisk_log_size":8205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:43.270299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382795776,"num_objects":5841,"num_object_clones":0,"num_object_copies":17523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9505,"num_write_kb":491072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":59,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.839167+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T15:02:59.839167+0000","last_peered":"2024-09-13T15:02:59.839167+0000","last_clean":"2024-09-13T15:02:59.839167+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T15:02:59.839167+0000","last_undegraded":"2024-09-13T15:02:59.839167+0000","last_fullsized":"2024-09-13T15:02:59.839167+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:20.897906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837509+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T15:02:59.837509+0000","last_peered":"2024-09-13T15:02:59.837509+0000","last_clean":"2024-09-13T15:02:59.837509+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T15:02:59.837509+0000","last_undegraded":"2024-09-13T15:02:59.837509+0000","last_fullsized":"2024-09-13T15:02:59.837509+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"369'9597","reported_seq":9651,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.264801+0000","last_change":"2024-09-13T15:00:58.195408+0000","last_active":"2024-09-13T15:03:06.264801+0000","last_peered":"2024-09-13T15:03:06.264801+0000","last_clean":"2024-09-13T15:03:06.264801+0000","last_became_active":"2024-09-13T15:00:58.195010+0000","last_became_peered":"2024-09-13T15:00:58.195010+0000","last_unstale":"2024-09-13T15:03:06.264801+0000","last_undegraded":"2024-09-13T15:03:06.264801+0000","last_fullsized":"2024-09-13T15:03:06.264801+0000","mapping_epoch":348,"log_start":"351'1000","ondisk_log_start":"351'1000","created":344,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:41.965793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383451136,"num_objects":5851,"num_object_clones":0,"num_object_copies":17553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9597,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":38,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841857+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:02:59.841857+0000","last_peered":"2024-09-13T15:02:59.841857+0000","last_clean":"2024-09-13T15:02:59.841857+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:02:59.841857+0000","last_undegraded":"2024-09-13T15:02:59.841857+0000","last_fullsized":"2024-09-13T15:02:59.841857+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":343,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841990+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T15:02:59.841990+0000","last_peered":"2024-09-13T15:02:59.841990+0000","last_clean":"2024-09-13T15:02:59.841990+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T15:02:59.841990+0000","last_undegraded":"2024-09-13T15:02:59.841990+0000","last_fullsized":"2024-09-13T15:02:59.841990+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"369'9644","reported_seq":9698,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.113259+0000","last_change":"2024-09-13T15:01:33.864621+0000","last_active":"2024-09-13T15:03:07.113259+0000","last_peered":"2024-09-13T15:03:07.113259+0000","last_clean":"2024-09-13T15:03:07.113259+0000","last_became_active":"2024-09-13T15:01:33.856103+0000","last_became_peered":"2024-09-13T15:01:33.856103+0000","last_unstale":"2024-09-13T15:03:07.113259+0000","last_undegraded":"2024-09-13T15:03:07.113259+0000","last_fullsized":"2024-09-13T15:03:07.113259+0000","mapping_epoch":354,"log_start":"352'1900","ondisk_log_start":"352'1900","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:04:34.307026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380633088,"num_objects":5808,"num_object_clones":0,"num_object_copies":17424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9644,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":129,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880099+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:03:03.880099+0000","last_peered":"2024-09-13T15:03:03.880099+0000","last_clean":"2024-09-13T15:03:03.880099+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:03:03.880099+0000","last_undegraded":"2024-09-13T15:03:03.880099+0000","last_fullsized":"2024-09-13T15:03:03.880099+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":199,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882605+0000","last_change":"2024-09-13T15:00:59.190451+0000","last_active":"2024-09-13T15:03:03.882605+0000","last_peered":"2024-09-13T15:03:03.882605+0000","last_clean":"2024-09-13T15:03:03.882605+0000","last_became_active":"2024-09-13T15:00:59.189511+0000","last_became_peered":"2024-09-13T15:00:59.189511+0000","last_unstale":"2024-09-13T15:03:03.882605+0000","last_undegraded":"2024-09-13T15:03:03.882605+0000","last_fullsized":"2024-09-13T15:03:03.882605+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:58.355252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"369'9596","reported_seq":9637,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.132832+0000","last_change":"2024-09-13T15:00:58.196823+0000","last_active":"2024-09-13T15:03:06.132832+0000","last_peered":"2024-09-13T15:03:06.132832+0000","last_clean":"2024-09-13T15:03:06.132832+0000","last_became_active":"2024-09-13T15:00:58.196522+0000","last_became_peered":"2024-09-13T15:00:58.196522+0000","last_unstale":"2024-09-13T15:03:06.132832+0000","last_undegraded":"2024-09-13T15:03:06.132832+0000","last_fullsized":"2024-09-13T15:03:06.132832+0000","mapping_epoch":348,"log_start":"353'2900","ondisk_log_start":"353'2900","created":344,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:09.740245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381157376,"num_objects":5816,"num_object_clones":0,"num_object_copies":17448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9596,"num_write_kb":493184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":916,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880532+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:03:03.880532+0000","last_peered":"2024-09-13T15:03:03.880532+0000","last_clean":"2024-09-13T15:03:03.880532+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:03:03.880532+0000","last_undegraded":"2024-09-13T15:03:03.880532+0000","last_fullsized":"2024-09-13T15:03:03.880532+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":628,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880750+0000","last_change":"2024-09-13T15:00:59.189216+0000","last_active":"2024-09-13T15:03:03.880750+0000","last_peered":"2024-09-13T15:03:03.880750+0000","last_clean":"2024-09-13T15:03:03.880750+0000","last_became_active":"2024-09-13T15:00:59.188828+0000","last_became_peered":"2024-09-13T15:00:59.188828+0000","last_unstale":"2024-09-13T15:03:03.880750+0000","last_undegraded":"2024-09-13T15:03:03.880750+0000","last_fullsized":"2024-09-13T15:03:03.880750+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:02.854311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837262+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:02:59.837262+0000","last_peered":"2024-09-13T15:02:59.837262+0000","last_clean":"2024-09-13T15:02:59.837262+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:02:59.837262+0000","last_undegraded":"2024-09-13T15:02:59.837262+0000","last_fullsized":"2024-09-13T15:02:59.837262+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"369'9412","reported_seq":9447,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.122917+0000","last_change":"2024-09-13T15:00:51.158719+0000","last_active":"2024-09-13T15:03:06.122917+0000","last_peered":"2024-09-13T15:03:06.122917+0000","last_clean":"2024-09-13T15:03:06.122917+0000","last_became_active":"2024-09-13T15:00:51.158433+0000","last_became_peered":"2024-09-13T15:00:51.158433+0000","last_unstale":"2024-09-13T15:03:06.122917+0000","last_undegraded":"2024-09-13T15:03:06.122917+0000","last_fullsized":"2024-09-13T15:03:06.122917+0000","mapping_epoch":344,"log_start":"351'1300","ondisk_log_start":"351'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8112,"log_dups_size":0,"ondisk_log_size":8112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:33.014048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374210560,"num_objects":5710,"num_object_clones":0,"num_object_copies":17130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9412,"num_write_kb":483904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":413,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.881766+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:03:03.881766+0000","last_peered":"2024-09-13T15:03:03.881766+0000","last_clean":"2024-09-13T15:03:03.881766+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:03:03.881766+0000","last_undegraded":"2024-09-13T15:03:03.881766+0000","last_fullsized":"2024-09-13T15:03:03.881766+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":391,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842110+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T15:02:59.842110+0000","last_peered":"2024-09-13T15:02:59.842110+0000","last_clean":"2024-09-13T15:02:59.842110+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T15:02:59.842110+0000","last_undegraded":"2024-09-13T15:02:59.842110+0000","last_fullsized":"2024-09-13T15:02:59.842110+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.838434+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T15:02:59.838434+0000","last_peered":"2024-09-13T15:02:59.838434+0000","last_clean":"2024-09-13T15:02:59.838434+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T15:02:59.838434+0000","last_undegraded":"2024-09-13T15:02:59.838434+0000","last_fullsized":"2024-09-13T15:02:59.838434+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"369'9427","reported_seq":9462,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.136060+0000","last_change":"2024-09-13T15:00:51.641016+0000","last_active":"2024-09-13T15:03:06.136060+0000","last_peered":"2024-09-13T15:03:06.136060+0000","last_clean":"2024-09-13T15:03:06.136060+0000","last_became_active":"2024-09-13T15:00:51.640705+0000","last_became_peered":"2024-09-13T15:00:51.640705+0000","last_unstale":"2024-09-13T15:03:06.136060+0000","last_undegraded":"2024-09-13T15:03:06.136060+0000","last_fullsized":"2024-09-13T15:03:06.136060+0000","mapping_epoch":344,"log_start":"351'1300","ondisk_log_start":"351'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8127,"log_dups_size":0,"ondisk_log_size":8127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:37.313904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372703232,"num_objects":5687,"num_object_clones":0,"num_object_copies":17061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9427,"num_write_kb":483648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837398+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:02:59.837398+0000","last_peered":"2024-09-13T15:02:59.837398+0000","last_clean":"2024-09-13T15:02:59.837398+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:02:59.837398+0000","last_undegraded":"2024-09-13T15:02:59.837398+0000","last_fullsized":"2024-09-13T15:02:59.837398+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.839230+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T15:02:59.839230+0000","last_peered":"2024-09-13T15:02:59.839230+0000","last_clean":"2024-09-13T15:02:59.839230+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T15:02:59.839230+0000","last_undegraded":"2024-09-13T15:02:59.839230+0000","last_fullsized":"2024-09-13T15:02:59.839230+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"369'9726","reported_seq":9760,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.169677+0000","last_change":"2024-09-13T15:00:51.159137+0000","last_active":"2024-09-13T15:03:06.169677+0000","last_peered":"2024-09-13T15:03:06.169677+0000","last_clean":"2024-09-13T15:03:06.169677+0000","last_became_active":"2024-09-13T15:00:51.158825+0000","last_became_peered":"2024-09-13T15:00:51.158825+0000","last_unstale":"2024-09-13T15:03:06.169677+0000","last_undegraded":"2024-09-13T15:03:06.169677+0000","last_fullsized":"2024-09-13T15:03:06.169677+0000","mapping_epoch":344,"log_start":"353'3000","ondisk_log_start":"353'3000","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:36.728759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388235264,"num_objects":5924,"num_object_clones":0,"num_object_copies":17772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9726,"num_write_kb":500800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":413,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.881392+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:03:03.881392+0000","last_peered":"2024-09-13T15:03:03.881392+0000","last_clean":"2024-09-13T15:03:03.881392+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:03:03.881392+0000","last_undegraded":"2024-09-13T15:03:03.881392+0000","last_fullsized":"2024-09-13T15:03:03.881392+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":567,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879769+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:03:03.879769+0000","last_peered":"2024-09-13T15:03:03.879769+0000","last_clean":"2024-09-13T15:03:03.879769+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:03:03.879769+0000","last_undegraded":"2024-09-13T15:03:03.879769+0000","last_fullsized":"2024-09-13T15:03:03.879769+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":580,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.872008+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:03:03.872008+0000","last_peered":"2024-09-13T15:03:03.872008+0000","last_clean":"2024-09-13T15:03:03.872008+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:03:03.872008+0000","last_undegraded":"2024-09-13T15:03:03.872008+0000","last_fullsized":"2024-09-13T15:03:03.872008+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"369'9497","reported_seq":9558,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:04.865234+0000","last_change":"2024-09-13T15:00:51.159165+0000","last_active":"2024-09-13T15:03:04.865234+0000","last_peered":"2024-09-13T15:03:04.865234+0000","last_clean":"2024-09-13T15:03:04.865234+0000","last_became_active":"2024-09-13T15:00:51.158866+0000","last_became_peered":"2024-09-13T15:00:51.158866+0000","last_unstale":"2024-09-13T15:03:04.865234+0000","last_undegraded":"2024-09-13T15:03:04.865234+0000","last_fullsized":"2024-09-13T15:03:04.865234+0000","mapping_epoch":344,"log_start":"351'1300","ondisk_log_start":"351'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:18.372131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380567552,"num_objects":5807,"num_object_clones":0,"num_object_copies":17421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9497,"num_write_kb":489728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":536,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882221+0000","last_change":"2024-09-13T15:00:58.193515+0000","last_active":"2024-09-13T15:03:03.882221+0000","last_peered":"2024-09-13T15:03:03.882221+0000","last_clean":"2024-09-13T15:03:03.882221+0000","last_became_active":"2024-09-13T15:00:58.193108+0000","last_became_peered":"2024-09-13T15:00:58.193108+0000","last_unstale":"2024-09-13T15:03:03.882221+0000","last_undegraded":"2024-09-13T15:03:03.882221+0000","last_fullsized":"2024-09-13T15:03:03.882221+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:30.120095+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":51,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883784+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:03:03.883784+0000","last_peered":"2024-09-13T15:03:03.883784+0000","last_clean":"2024-09-13T15:03:03.883784+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:03:03.883784+0000","last_undegraded":"2024-09-13T15:03:03.883784+0000","last_fullsized":"2024-09-13T15:03:03.883784+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":552,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879692+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:03:03.879692+0000","last_peered":"2024-09-13T15:03:03.879692+0000","last_clean":"2024-09-13T15:03:03.879692+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:03:03.879692+0000","last_undegraded":"2024-09-13T15:03:03.879692+0000","last_fullsized":"2024-09-13T15:03:03.879692+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":353,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841789+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:02:59.841789+0000","last_peered":"2024-09-13T15:02:59.841789+0000","last_clean":"2024-09-13T15:02:59.841789+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:02:59.841789+0000","last_undegraded":"2024-09-13T15:02:59.841789+0000","last_fullsized":"2024-09-13T15:02:59.841789+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"369'9592","reported_seq":9625,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.147665+0000","last_change":"2024-09-13T15:00:51.640917+0000","last_active":"2024-09-13T15:03:06.147665+0000","last_peered":"2024-09-13T15:03:06.147665+0000","last_clean":"2024-09-13T15:03:06.147665+0000","last_became_active":"2024-09-13T15:00:51.640598+0000","last_became_peered":"2024-09-13T15:00:51.640598+0000","last_unstale":"2024-09-13T15:03:06.147665+0000","last_undegraded":"2024-09-13T15:03:06.147665+0000","last_fullsized":"2024-09-13T15:03:06.147665+0000","mapping_epoch":344,"log_start":"352'1400","ondisk_log_start":"352'1400","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:45.675640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378535936,"num_objects":5776,"num_object_clones":0,"num_object_copies":17328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9592,"num_write_kb":491776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":847,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.871810+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:03:03.871810+0000","last_peered":"2024-09-13T15:03:03.871810+0000","last_clean":"2024-09-13T15:03:03.871810+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:03:03.871810+0000","last_undegraded":"2024-09-13T15:03:03.871810+0000","last_fullsized":"2024-09-13T15:03:03.871810+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":564,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.875639+0000","last_change":"2024-09-13T15:00:58.198644+0000","last_active":"2024-09-13T15:03:03.875639+0000","last_peered":"2024-09-13T15:03:03.875639+0000","last_clean":"2024-09-13T15:03:03.875639+0000","last_became_active":"2024-09-13T15:00:58.198334+0000","last_became_peered":"2024-09-13T15:00:58.198334+0000","last_unstale":"2024-09-13T15:03:03.875639+0000","last_undegraded":"2024-09-13T15:03:03.875639+0000","last_fullsized":"2024-09-13T15:03:03.875639+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:52.461192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882446+0000","last_change":"2024-09-13T15:00:59.188848+0000","last_active":"2024-09-13T15:03:03.882446+0000","last_peered":"2024-09-13T15:03:03.882446+0000","last_clean":"2024-09-13T15:03:03.882446+0000","last_became_active":"2024-09-13T15:00:59.188521+0000","last_became_peered":"2024-09-13T15:00:59.188521+0000","last_unstale":"2024-09-13T15:03:03.882446+0000","last_undegraded":"2024-09-13T15:03:03.882446+0000","last_fullsized":"2024-09-13T15:03:03.882446+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:45.161336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":66,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883201+0000","last_change":"2024-09-13T15:00:58.196782+0000","last_active":"2024-09-13T15:03:03.883201+0000","last_peered":"2024-09-13T15:03:03.883201+0000","last_clean":"2024-09-13T15:03:03.883201+0000","last_became_active":"2024-09-13T15:00:58.194669+0000","last_became_peered":"2024-09-13T15:00:58.194669+0000","last_unstale":"2024-09-13T15:03:03.883201+0000","last_undegraded":"2024-09-13T15:03:03.883201+0000","last_fullsized":"2024-09-13T15:03:03.883201+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:56.299128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"369'9600","reported_seq":9642,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.106167+0000","last_change":"2024-09-13T15:01:33.862631+0000","last_active":"2024-09-13T15:03:06.106167+0000","last_peered":"2024-09-13T15:03:06.106167+0000","last_clean":"2024-09-13T15:03:06.106167+0000","last_became_active":"2024-09-13T15:01:33.855592+0000","last_became_peered":"2024-09-13T15:01:33.855592+0000","last_unstale":"2024-09-13T15:03:06.106167+0000","last_undegraded":"2024-09-13T15:03:06.106167+0000","last_fullsized":"2024-09-13T15:03:06.106167+0000","mapping_epoch":354,"log_start":"353'2900","ondisk_log_start":"353'2900","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6700,"log_dups_size":0,"ondisk_log_size":6700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:33:37.590412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380239872,"num_objects":5802,"num_object_clones":0,"num_object_copies":17406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9600,"num_write_kb":492864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":611,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882222+0000","last_change":"2024-09-13T15:00:59.187059+0000","last_active":"2024-09-13T15:03:03.882222+0000","last_peered":"2024-09-13T15:03:03.882222+0000","last_clean":"2024-09-13T15:03:03.882222+0000","last_became_active":"2024-09-13T15:00:59.186471+0000","last_became_peered":"2024-09-13T15:00:59.186471+0000","last_unstale":"2024-09-13T15:03:03.882222+0000","last_undegraded":"2024-09-13T15:03:03.882222+0000","last_fullsized":"2024-09-13T15:03:03.882222+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:15.251496+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":616,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.881966+0000","last_change":"2024-09-13T15:00:58.191231+0000","last_active":"2024-09-13T15:03:03.881966+0000","last_peered":"2024-09-13T15:03:03.881966+0000","last_clean":"2024-09-13T15:03:03.881966+0000","last_became_active":"2024-09-13T15:00:58.190874+0000","last_became_peered":"2024-09-13T15:00:58.190874+0000","last_unstale":"2024-09-13T15:03:03.881966+0000","last_undegraded":"2024-09-13T15:03:03.881966+0000","last_fullsized":"2024-09-13T15:03:03.881966+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:42.995687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":565,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.875804+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:03:03.875804+0000","last_peered":"2024-09-13T15:03:03.875804+0000","last_clean":"2024-09-13T15:03:03.875804+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:03:03.875804+0000","last_undegraded":"2024-09-13T15:03:03.875804+0000","last_fullsized":"2024-09-13T15:03:03.875804+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":626,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.871876+0000","last_change":"2024-09-13T15:00:58.189504+0000","last_active":"2024-09-13T15:03:03.871876+0000","last_peered":"2024-09-13T15:03:03.871876+0000","last_clean":"2024-09-13T15:03:03.871876+0000","last_became_active":"2024-09-13T15:00:58.189158+0000","last_became_peered":"2024-09-13T15:00:58.189158+0000","last_unstale":"2024-09-13T15:03:03.871876+0000","last_undegraded":"2024-09-13T15:03:03.871876+0000","last_fullsized":"2024-09-13T15:03:03.871876+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:23.156063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"369'9665","reported_seq":9700,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.170558+0000","last_change":"2024-09-13T15:00:51.638684+0000","last_active":"2024-09-13T15:03:06.170558+0000","last_peered":"2024-09-13T15:03:06.170558+0000","last_clean":"2024-09-13T15:03:06.170558+0000","last_became_active":"2024-09-13T15:00:51.638349+0000","last_became_peered":"2024-09-13T15:00:51.638349+0000","last_unstale":"2024-09-13T15:03:06.170558+0000","last_undegraded":"2024-09-13T15:03:06.170558+0000","last_fullsized":"2024-09-13T15:03:06.170558+0000","mapping_epoch":344,"log_start":"352'2900","ondisk_log_start":"352'2900","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:53:23.643248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385089560,"num_objects":5877,"num_object_clones":0,"num_object_copies":17631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5877,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9665,"num_write_kb":497281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":547,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882593+0000","last_change":"2024-09-13T15:01:33.848879+0000","last_active":"2024-09-13T15:03:03.882593+0000","last_peered":"2024-09-13T15:03:03.882593+0000","last_clean":"2024-09-13T15:03:03.882593+0000","last_became_active":"2024-09-13T15:01:33.848475+0000","last_became_peered":"2024-09-13T15:01:33.848475+0000","last_unstale":"2024-09-13T15:03:03.882593+0000","last_undegraded":"2024-09-13T15:03:03.882593+0000","last_fullsized":"2024-09-13T15:03:03.882593+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:46.934186+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877343+0000","last_change":"2024-09-13T15:01:33.852433+0000","last_active":"2024-09-13T15:03:03.877343+0000","last_peered":"2024-09-13T15:03:03.877343+0000","last_clean":"2024-09-13T15:03:03.877343+0000","last_became_active":"2024-09-13T15:01:33.852015+0000","last_became_peered":"2024-09-13T15:01:33.852015+0000","last_unstale":"2024-09-13T15:03:03.877343+0000","last_undegraded":"2024-09-13T15:03:03.877343+0000","last_fullsized":"2024-09-13T15:03:03.877343+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:52:03.246939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":66,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837480+0000","last_change":"2024-09-13T15:00:59.305010+0000","last_active":"2024-09-13T15:02:59.837480+0000","last_peered":"2024-09-13T15:02:59.837480+0000","last_clean":"2024-09-13T15:02:59.837480+0000","last_became_active":"2024-09-13T15:00:59.304744+0000","last_became_peered":"2024-09-13T15:00:59.304744+0000","last_unstale":"2024-09-13T15:02:59.837480+0000","last_undegraded":"2024-09-13T15:02:59.837480+0000","last_fullsized":"2024-09-13T15:02:59.837480+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:58:57.917883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":578,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876037+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:03:03.876037+0000","last_peered":"2024-09-13T15:03:03.876037+0000","last_clean":"2024-09-13T15:03:03.876037+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:03:03.876037+0000","last_undegraded":"2024-09-13T15:03:03.876037+0000","last_fullsized":"2024-09-13T15:03:03.876037+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"369'9698","reported_seq":9753,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.150157+0000","last_change":"2024-09-13T15:01:33.863073+0000","last_active":"2024-09-13T15:03:07.150157+0000","last_peered":"2024-09-13T15:03:07.150157+0000","last_clean":"2024-09-13T15:03:07.150157+0000","last_became_active":"2024-09-13T15:01:33.855365+0000","last_became_peered":"2024-09-13T15:01:33.855365+0000","last_unstale":"2024-09-13T15:03:07.150157+0000","last_undegraded":"2024-09-13T15:03:07.150157+0000","last_fullsized":"2024-09-13T15:03:07.150157+0000","mapping_epoch":354,"log_start":"352'2000","ondisk_log_start":"352'2000","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":7698,"log_dups_size":0,"ondisk_log_size":7698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:18.315313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379715584,"num_objects":5794,"num_object_clones":0,"num_object_copies":17382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9698,"num_write_kb":495744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":620,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842476+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:02:59.842476+0000","last_peered":"2024-09-13T15:02:59.842476+0000","last_clean":"2024-09-13T15:02:59.842476+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:02:59.842476+0000","last_undegraded":"2024-09-13T15:02:59.842476+0000","last_fullsized":"2024-09-13T15:02:59.842476+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880409+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:03:03.880409+0000","last_peered":"2024-09-13T15:03:03.880409+0000","last_clean":"2024-09-13T15:03:03.880409+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:03:03.880409+0000","last_undegraded":"2024-09-13T15:03:03.880409+0000","last_fullsized":"2024-09-13T15:03:03.880409+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":641,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876391+0000","last_change":"2024-09-13T15:00:58.197235+0000","last_active":"2024-09-13T15:03:03.876391+0000","last_peered":"2024-09-13T15:03:03.876391+0000","last_clean":"2024-09-13T15:03:03.876391+0000","last_became_active":"2024-09-13T15:00:58.196938+0000","last_became_peered":"2024-09-13T15:00:58.196938+0000","last_unstale":"2024-09-13T15:03:03.876391+0000","last_undegraded":"2024-09-13T15:03:03.876391+0000","last_fullsized":"2024-09-13T15:03:03.876391+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:33:33.195527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":504,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876679+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:03:03.876679+0000","last_peered":"2024-09-13T15:03:03.876679+0000","last_clean":"2024-09-13T15:03:03.876679+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:03:03.876679+0000","last_undegraded":"2024-09-13T15:03:03.876679+0000","last_fullsized":"2024-09-13T15:03:03.876679+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"369'9731","reported_seq":9793,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.516646+0000","last_change":"2024-09-13T15:00:51.158952+0000","last_active":"2024-09-13T15:03:07.516646+0000","last_peered":"2024-09-13T15:03:07.516646+0000","last_clean":"2024-09-13T15:03:07.516646+0000","last_became_active":"2024-09-13T15:00:51.158650+0000","last_became_peered":"2024-09-13T15:00:51.158650+0000","last_unstale":"2024-09-13T15:03:07.516646+0000","last_undegraded":"2024-09-13T15:03:07.516646+0000","last_fullsized":"2024-09-13T15:03:07.516646+0000","mapping_epoch":344,"log_start":"351'1100","ondisk_log_start":"351'1100","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8631,"log_dups_size":0,"ondisk_log_size":8631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:58.684962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375980032,"num_objects":5737,"num_object_clones":0,"num_object_copies":17211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9731,"num_write_kb":494976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":669,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841945+0000","last_change":"2024-09-13T15:00:58.199832+0000","last_active":"2024-09-13T15:02:59.841945+0000","last_peered":"2024-09-13T15:02:59.841945+0000","last_clean":"2024-09-13T15:02:59.841945+0000","last_became_active":"2024-09-13T15:00:58.199563+0000","last_became_peered":"2024-09-13T15:00:58.199563+0000","last_unstale":"2024-09-13T15:02:59.841945+0000","last_undegraded":"2024-09-13T15:02:59.841945+0000","last_fullsized":"2024-09-13T15:02:59.841945+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:58.757475+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":565,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876165+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:03:03.876165+0000","last_peered":"2024-09-13T15:03:03.876165+0000","last_clean":"2024-09-13T15:03:03.876165+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:03:03.876165+0000","last_undegraded":"2024-09-13T15:03:03.876165+0000","last_fullsized":"2024-09-13T15:03:03.876165+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":201,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.882687+0000","last_change":"2024-09-13T15:00:59.185867+0000","last_active":"2024-09-13T15:03:03.882687+0000","last_peered":"2024-09-13T15:03:03.882687+0000","last_clean":"2024-09-13T15:03:03.882687+0000","last_became_active":"2024-09-13T15:00:59.184841+0000","last_became_peered":"2024-09-13T15:00:59.184841+0000","last_unstale":"2024-09-13T15:03:03.882687+0000","last_undegraded":"2024-09-13T15:03:03.882687+0000","last_fullsized":"2024-09-13T15:03:03.882687+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:58.991299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883969+0000","last_change":"2024-09-13T15:00:58.192299+0000","last_active":"2024-09-13T15:03:03.883969+0000","last_peered":"2024-09-13T15:03:03.883969+0000","last_clean":"2024-09-13T15:03:03.883969+0000","last_became_active":"2024-09-13T15:00:58.191940+0000","last_became_peered":"2024-09-13T15:00:58.191940+0000","last_unstale":"2024-09-13T15:03:03.883969+0000","last_undegraded":"2024-09-13T15:03:03.883969+0000","last_fullsized":"2024-09-13T15:03:03.883969+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:31.247758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"369'9701","reported_seq":9764,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:06.264276+0000","last_change":"2024-09-13T15:01:33.853395+0000","last_active":"2024-09-13T15:03:06.264276+0000","last_peered":"2024-09-13T15:03:06.264276+0000","last_clean":"2024-09-13T15:03:06.264276+0000","last_became_active":"2024-09-13T15:01:33.849065+0000","last_became_peered":"2024-09-13T15:01:33.849065+0000","last_unstale":"2024-09-13T15:03:06.264276+0000","last_undegraded":"2024-09-13T15:03:06.264276+0000","last_fullsized":"2024-09-13T15:03:06.264276+0000","mapping_epoch":354,"log_start":"351'1100","ondisk_log_start":"351'1100","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8601,"log_dups_size":0,"ondisk_log_size":8601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:01.268141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385548288,"num_objects":5883,"num_object_clones":0,"num_object_copies":17649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9701,"num_write_kb":498688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":548,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.838600+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:02:59.838600+0000","last_peered":"2024-09-13T15:02:59.838600+0000","last_clean":"2024-09-13T15:02:59.838600+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:02:59.838600+0000","last_undegraded":"2024-09-13T15:02:59.838600+0000","last_fullsized":"2024-09-13T15:02:59.838600+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":586,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880413+0000","last_change":"2024-09-13T15:00:58.198098+0000","last_active":"2024-09-13T15:03:03.880413+0000","last_peered":"2024-09-13T15:03:03.880413+0000","last_clean":"2024-09-13T15:03:03.880413+0000","last_became_active":"2024-09-13T15:00:58.197740+0000","last_became_peered":"2024-09-13T15:00:58.197740+0000","last_unstale":"2024-09-13T15:03:03.880413+0000","last_undegraded":"2024-09-13T15:03:03.880413+0000","last_fullsized":"2024-09-13T15:03:03.880413+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:20.126199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":579,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.872160+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:03:03.872160+0000","last_peered":"2024-09-13T15:03:03.872160+0000","last_clean":"2024-09-13T15:03:03.872160+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:03:03.872160+0000","last_undegraded":"2024-09-13T15:03:03.872160+0000","last_fullsized":"2024-09-13T15:03:03.872160+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":590,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879590+0000","last_change":"2024-09-13T15:00:58.198790+0000","last_active":"2024-09-13T15:03:03.879590+0000","last_peered":"2024-09-13T15:03:03.879590+0000","last_clean":"2024-09-13T15:03:03.879590+0000","last_became_active":"2024-09-13T15:00:58.198508+0000","last_became_peered":"2024-09-13T15:00:58.198508+0000","last_unstale":"2024-09-13T15:03:03.879590+0000","last_undegraded":"2024-09-13T15:03:03.879590+0000","last_fullsized":"2024-09-13T15:03:03.879590+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:50:51.003072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"369'9897","reported_seq":9959,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:07.536079+0000","last_change":"2024-09-13T15:00:51.157239+0000","last_active":"2024-09-13T15:03:07.536079+0000","last_peered":"2024-09-13T15:03:07.536079+0000","last_clean":"2024-09-13T15:03:07.536079+0000","last_became_active":"2024-09-13T15:00:51.156946+0000","last_became_peered":"2024-09-13T15:00:51.156946+0000","last_unstale":"2024-09-13T15:03:07.536079+0000","last_undegraded":"2024-09-13T15:03:07.536079+0000","last_fullsized":"2024-09-13T15:03:07.536079+0000","mapping_epoch":344,"log_start":"351'1300","ondisk_log_start":"351'1300","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:54.326025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385941504,"num_objects":5889,"num_object_clones":0,"num_object_copies":17667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9897,"num_write_kb":505152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":826,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879827+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:03:03.879827+0000","last_peered":"2024-09-13T15:03:03.879827+0000","last_clean":"2024-09-13T15:03:03.879827+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:03:03.879827+0000","last_undegraded":"2024-09-13T15:03:03.879827+0000","last_fullsized":"2024-09-13T15:03:03.879827+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":67,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883803+0000","last_change":"2024-09-13T15:00:59.187967+0000","last_active":"2024-09-13T15:03:03.883803+0000","last_peered":"2024-09-13T15:03:03.883803+0000","last_clean":"2024-09-13T15:03:03.883803+0000","last_became_active":"2024-09-13T15:00:59.187518+0000","last_became_peered":"2024-09-13T15:00:59.187518+0000","last_unstale":"2024-09-13T15:03:03.883803+0000","last_undegraded":"2024-09-13T15:03:03.883803+0000","last_fullsized":"2024-09-13T15:03:03.883803+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:14:59.602191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":357,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842623+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:02:59.842623+0000","last_peered":"2024-09-13T15:02:59.842623+0000","last_clean":"2024-09-13T15:02:59.842623+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:02:59.842623+0000","last_undegraded":"2024-09-13T15:02:59.842623+0000","last_fullsized":"2024-09-13T15:02:59.842623+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837067+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:02:59.837067+0000","last_peered":"2024-09-13T15:02:59.837067+0000","last_clean":"2024-09-13T15:02:59.837067+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:02:59.837067+0000","last_undegraded":"2024-09-13T15:02:59.837067+0000","last_fullsized":"2024-09-13T15:02:59.837067+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837551+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T15:02:59.837551+0000","last_peered":"2024-09-13T15:02:59.837551+0000","last_clean":"2024-09-13T15:02:59.837551+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T15:02:59.837551+0000","last_undegraded":"2024-09-13T15:02:59.837551+0000","last_fullsized":"2024-09-13T15:02:59.837551+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":65,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837193+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:02:59.837193+0000","last_peered":"2024-09-13T15:02:59.837193+0000","last_clean":"2024-09-13T15:02:59.837193+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:02:59.837193+0000","last_undegraded":"2024-09-13T15:02:59.837193+0000","last_fullsized":"2024-09-13T15:02:59.837193+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":485,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877469+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:03:03.877469+0000","last_peered":"2024-09-13T15:03:03.877469+0000","last_clean":"2024-09-13T15:03:03.877469+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:03:03.877469+0000","last_undegraded":"2024-09-13T15:03:03.877469+0000","last_fullsized":"2024-09-13T15:03:03.877469+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":540,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883306+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:03:03.883306+0000","last_peered":"2024-09-13T15:03:03.883306+0000","last_clean":"2024-09-13T15:03:03.883306+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:03:03.883306+0000","last_undegraded":"2024-09-13T15:03:03.883306+0000","last_fullsized":"2024-09-13T15:03:03.883306+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.872237+0000","last_change":"2024-09-13T15:01:33.853899+0000","last_active":"2024-09-13T15:03:03.872237+0000","last_peered":"2024-09-13T15:03:03.872237+0000","last_clean":"2024-09-13T15:03:03.872237+0000","last_became_active":"2024-09-13T15:01:33.853515+0000","last_became_peered":"2024-09-13T15:01:33.853515+0000","last_unstale":"2024-09-13T15:03:03.872237+0000","last_undegraded":"2024-09-13T15:03:03.872237+0000","last_fullsized":"2024-09-13T15:03:03.872237+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:04.035329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":89,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.841502+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:02:59.841502+0000","last_peered":"2024-09-13T15:02:59.841502+0000","last_clean":"2024-09-13T15:02:59.841502+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:02:59.841502+0000","last_undegraded":"2024-09-13T15:02:59.841502+0000","last_fullsized":"2024-09-13T15:02:59.841502+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883260+0000","last_change":"2024-09-13T15:00:58.194381+0000","last_active":"2024-09-13T15:03:03.883260+0000","last_peered":"2024-09-13T15:03:03.883260+0000","last_clean":"2024-09-13T15:03:03.883260+0000","last_became_active":"2024-09-13T15:00:58.194073+0000","last_became_peered":"2024-09-13T15:00:58.194073+0000","last_unstale":"2024-09-13T15:03:03.883260+0000","last_undegraded":"2024-09-13T15:03:03.883260+0000","last_fullsized":"2024-09-13T15:03:03.883260+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:00.089072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":564,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879503+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:03:03.879503+0000","last_peered":"2024-09-13T15:03:03.879503+0000","last_clean":"2024-09-13T15:03:03.879503+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:03:03.879503+0000","last_undegraded":"2024-09-13T15:03:03.879503+0000","last_fullsized":"2024-09-13T15:03:03.879503+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":344,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842551+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T15:02:59.842551+0000","last_peered":"2024-09-13T15:02:59.842551+0000","last_clean":"2024-09-13T15:02:59.842551+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T15:02:59.842551+0000","last_undegraded":"2024-09-13T15:02:59.842551+0000","last_fullsized":"2024-09-13T15:02:59.842551+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":619,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.879672+0000","last_change":"2024-09-13T15:00:58.198846+0000","last_active":"2024-09-13T15:03:03.879672+0000","last_peered":"2024-09-13T15:03:03.879672+0000","last_clean":"2024-09-13T15:03:03.879672+0000","last_became_active":"2024-09-13T15:00:58.198544+0000","last_became_peered":"2024-09-13T15:00:58.198544+0000","last_unstale":"2024-09-13T15:03:03.879672+0000","last_undegraded":"2024-09-13T15:03:03.879672+0000","last_fullsized":"2024-09-13T15:03:03.879672+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:06.715499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":151,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883024+0000","last_change":"2024-09-13T15:00:58.197368+0000","last_active":"2024-09-13T15:03:03.883024+0000","last_peered":"2024-09-13T15:03:03.883024+0000","last_clean":"2024-09-13T15:03:03.883024+0000","last_became_active":"2024-09-13T15:00:58.196459+0000","last_became_peered":"2024-09-13T15:00:58.196459+0000","last_unstale":"2024-09-13T15:03:03.883024+0000","last_undegraded":"2024-09-13T15:03:03.883024+0000","last_fullsized":"2024-09-13T15:03:03.883024+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:58.563761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":577,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.871679+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:03:03.871679+0000","last_peered":"2024-09-13T15:03:03.871679+0000","last_clean":"2024-09-13T15:03:03.871679+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:03:03.871679+0000","last_undegraded":"2024-09-13T15:03:03.871679+0000","last_fullsized":"2024-09-13T15:03:03.871679+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":578,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880206+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:03:03.880206+0000","last_peered":"2024-09-13T15:03:03.880206+0000","last_clean":"2024-09-13T15:03:03.880206+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:03:03.880206+0000","last_undegraded":"2024-09-13T15:03:03.880206+0000","last_fullsized":"2024-09-13T15:03:03.880206+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842738+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T15:02:59.842738+0000","last_peered":"2024-09-13T15:02:59.842738+0000","last_clean":"2024-09-13T15:02:59.842738+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T15:02:59.842738+0000","last_undegraded":"2024-09-13T15:02:59.842738+0000","last_fullsized":"2024-09-13T15:02:59.842738+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:59.608331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":590,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.884142+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:03:03.884142+0000","last_peered":"2024-09-13T15:03:03.884142+0000","last_clean":"2024-09-13T15:03:03.884142+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:03:03.884142+0000","last_undegraded":"2024-09-13T15:03:03.884142+0000","last_fullsized":"2024-09-13T15:03:03.884142+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":62,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883872+0000","last_change":"2024-09-13T15:00:59.191425+0000","last_active":"2024-09-13T15:03:03.883872+0000","last_peered":"2024-09-13T15:03:03.883872+0000","last_clean":"2024-09-13T15:03:03.883872+0000","last_became_active":"2024-09-13T15:00:59.189942+0000","last_became_peered":"2024-09-13T15:00:59.189942+0000","last_unstale":"2024-09-13T15:03:03.883872+0000","last_undegraded":"2024-09-13T15:03:03.883872+0000","last_fullsized":"2024-09-13T15:03:03.883872+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:45.702309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837356+0000","last_change":"2024-09-13T14:59:12.712394+0000","last_active":"2024-09-13T15:02:59.837356+0000","last_peered":"2024-09-13T15:02:59.837356+0000","last_clean":"2024-09-13T15:02:59.837356+0000","last_became_active":"2024-09-13T14:59:12.711381+0000","last_became_peered":"2024-09-13T14:59:12.711381+0000","last_unstale":"2024-09-13T15:02:59.837356+0000","last_undegraded":"2024-09-13T15:02:59.837356+0000","last_fullsized":"2024-09-13T15:02:59.837356+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:39:28.567717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":337,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.837100+0000","last_change":"2024-09-13T15:01:33.852507+0000","last_active":"2024-09-13T15:02:59.837100+0000","last_peered":"2024-09-13T15:02:59.837100+0000","last_clean":"2024-09-13T15:02:59.837100+0000","last_became_active":"2024-09-13T15:01:33.852196+0000","last_became_peered":"2024-09-13T15:01:33.852196+0000","last_unstale":"2024-09-13T15:02:59.837100+0000","last_undegraded":"2024-09-13T15:02:59.837100+0000","last_fullsized":"2024-09-13T15:02:59.837100+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:28.217977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.842450+0000","last_change":"2024-09-13T15:00:59.184678+0000","last_active":"2024-09-13T15:02:59.842450+0000","last_peered":"2024-09-13T15:02:59.842450+0000","last_clean":"2024-09-13T15:02:59.842450+0000","last_became_active":"2024-09-13T15:00:59.184456+0000","last_became_peered":"2024-09-13T15:00:59.184456+0000","last_unstale":"2024-09-13T15:02:59.842450+0000","last_undegraded":"2024-09-13T15:02:59.842450+0000","last_fullsized":"2024-09-13T15:02:59.842450+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:01.011514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":322,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.838505+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:02:59.838505+0000","last_peered":"2024-09-13T15:02:59.838505+0000","last_clean":"2024-09-13T15:02:59.838505+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:02:59.838505+0000","last_undegraded":"2024-09-13T15:02:59.838505+0000","last_fullsized":"2024-09-13T15:02:59.838505+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":621,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880645+0000","last_change":"2024-09-13T15:00:58.197037+0000","last_active":"2024-09-13T15:03:03.880645+0000","last_peered":"2024-09-13T15:03:03.880645+0000","last_clean":"2024-09-13T15:03:03.880645+0000","last_became_active":"2024-09-13T15:00:58.196717+0000","last_became_peered":"2024-09-13T15:00:58.196717+0000","last_unstale":"2024-09-13T15:03:03.880645+0000","last_undegraded":"2024-09-13T15:03:03.880645+0000","last_fullsized":"2024-09-13T15:03:03.880645+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:43.954736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":587,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.884791+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:03:03.884791+0000","last_peered":"2024-09-13T15:03:03.884791+0000","last_clean":"2024-09-13T15:03:03.884791+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:03:03.884791+0000","last_undegraded":"2024-09-13T15:03:03.884791+0000","last_fullsized":"2024-09-13T15:03:03.884791+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":55,"reported_epoch":368,"state":"active+clean","last_fresh":"2024-09-13T15:02:59.836942+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T15:02:59.836942+0000","last_peered":"2024-09-13T15:02:59.836942+0000","last_clean":"2024-09-13T15:02:59.836942+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T15:02:59.836942+0000","last_undegraded":"2024-09-13T15:02:59.836942+0000","last_fullsized":"2024-09-13T15:02:59.836942+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":72,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883355+0000","last_change":"2024-09-13T15:00:58.194182+0000","last_active":"2024-09-13T15:03:03.883355+0000","last_peered":"2024-09-13T15:03:03.883355+0000","last_clean":"2024-09-13T15:03:03.883355+0000","last_became_active":"2024-09-13T15:00:58.193479+0000","last_became_peered":"2024-09-13T15:00:58.193479+0000","last_unstale":"2024-09-13T15:03:03.883355+0000","last_undegraded":"2024-09-13T15:03:03.883355+0000","last_fullsized":"2024-09-13T15:03:03.883355+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:34.966483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":6101991448,"num_objects":93110,"num_object_clones":0,"num_object_copies":279330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93110,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153888,"num_write_kb":7903873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18296811520,"data_stored":18296799304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125788,"ondisk_log_size":125788,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":5182,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234516,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3183024,"kb_used_data":1872228,"kb_used_omap":1,"kb_used_meta":1310718,"kb_avail":90562128,"statfs":{"total":95995035648,"available":92735619072,"internally_reserved":0,"allocated":1917161472,"data_stored":1915142207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1342175691},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496154,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3011108,"kb_used_data":1865116,"kb_used_omap":1,"kb_used_meta":1145982,"kb_avail":90734044,"statfs":{"total":95995035648,"available":92911661056,"internally_reserved":0,"allocated":1909878784,"data_stored":1907867650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1173486023},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757792,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4351264,"kb_used_data":3026196,"kb_used_omap":1,"kb_used_meta":1324862,"kb_avail":89393888,"statfs":{"total":95995035648,"available":91539341312,"internally_reserved":0,"allocated":3098824704,"data_stored":3096817801,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1356659142},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019428,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4474668,"kb_used_data":3384420,"kb_used_omap":1,"kb_used_meta":1090174,"kb_avail":89270484,"statfs":{"total":95995035648,"available":91412975616,"internally_reserved":0,"allocated":3465646080,"data_stored":3463626835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1116338630},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650640,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2399000,"kb_used_data":1506832,"kb_used_omap":4,"kb_used_meta":892155,"kb_avail":91346152,"statfs":{"total":95995035648,"available":93538459648,"internally_reserved":0,"allocated":1542995968,"data_stored":1540992923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":913567279},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879575,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4212832,"kb_used_data":3317080,"kb_used_omap":6,"kb_used_meta":895737,"kb_avail":89532320,"statfs":{"total":95995035648,"available":91681095680,"internally_reserved":0,"allocated":3396689920,"data_stored":3394679299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":917235257},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435591,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2980240,"kb_used_data":1887432,"kb_used_omap":5,"kb_used_meta":1092794,"kb_avail":90764912,"statfs":{"total":95995035648,"available":92943269888,"internally_reserved":0,"allocated":1932730368,"data_stored":1930734378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1119021409},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":348,"seq":1494648619035,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1591768,"kb_used_data":1483984,"kb_used_omap":10,"kb_used_meta":107765,"kb_avail":92153384,"statfs":{"total":95995035648,"available":94365065216,"internally_reserved":0,"allocated":1519599616,"data_stored":1517596541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10675,"internal_metadata":110351949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1149698048,"data_stored":1149698048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1919225856,"data_stored":1919221784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3385327616,"data_stored":3385327616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1524957184,"data_stored":1524957184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3443527680,"data_stored":3443523608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3072983040,"data_stored":3072983040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1903820800,"data_stored":1903820800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1897271296,"data_stored":1897267224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:03:13.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:03:13.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:13.963 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:13.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:13.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:14.065 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:13 smithi042 ceph-mon[113926]: pgmap v1333: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-13T15:03:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:13 smithi042 ceph-mon[108570]: pgmap v1333: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-13T15:03:14.123 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:14.225 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:13 smithi159 ceph-mon[102519]: pgmap v1333: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 928 op/s 2024-09-13T15:03:14.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:14.359 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:14.382 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:14.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:14.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:14.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:14.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:14.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:14.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:14.788 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:14.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:14.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:14.921 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:14.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:15.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:15.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:03:15.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:15.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:15.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.191 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:15.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:15.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:15.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:15.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:15.432 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:15.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:15.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:15.604 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:15.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:15.706 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:15.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:15.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:15.876 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:15.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:15.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:16.008 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:16.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:16.139 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:16.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:15 smithi159 ceph-mon[102519]: pgmap v1334: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:03:16.279 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:15 smithi042 ceph-mon[113926]: pgmap v1334: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:03:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:15 smithi042 ceph-mon[108570]: pgmap v1334: 99 pgs: 99 active+clean; 5.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-13T15:03:16.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:16.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:16.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:16.520 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:16.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:16.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:16.695 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:16.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:16.827 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:16.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:16.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:16.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:17.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:17.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:17.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:17.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:17.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.347 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.392 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:17.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:17.493 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:17.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.660 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:17.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.761 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:17.800 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:17.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:17.902 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:17.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:17.924 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:17.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:18.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:18.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:18.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:18.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:18.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:17 smithi159 ceph-mon[102519]: pgmap v1335: 99 pgs: 99 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:03:18.302 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:18.340 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:18.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:18.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:17 smithi042 ceph-mon[113926]: pgmap v1335: 99 pgs: 99 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:03:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:17 smithi042 ceph-mon[108570]: pgmap v1335: 99 pgs: 99 active+clean; 5.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-09-13T15:03:18.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:18.480 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:18.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:18.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:18.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:18.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:18.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:18.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:18.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:18.897 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:18.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:18.999 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:19.020 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:19.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:19.122 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:19.154 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:19.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:19.255 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:19.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:19.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:19.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:19.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:19.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.577 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:19.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:19.678 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:19.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:19.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:19.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:19.952 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:19.960 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:19 smithi042 ceph-mon[108570]: pgmap v1336: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-13T15:03:19.960 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:19] "GET /metrics HTTP/1.1" 200 36532 "" "Prometheus/2.51.0" 2024-09-13T15:03:19.960 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:19 smithi042 ceph-mon[113926]: pgmap v1336: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-13T15:03:19.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:19.992 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:19.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:20.093 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:20.131 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:19 smithi159 ceph-mon[102519]: pgmap v1336: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-13T15:03:20.295 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:03:20.295 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:03:20.295 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:03:20.295 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:03:20.295 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:03:20.343 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:20.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:20.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:20.528 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:20.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:20.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:20.630 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:20.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.667 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:20.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:20.768 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:20.802 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:20.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:20.920 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3736842097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:20.920 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:20.920 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:20.920 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:03:20.920 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3736842097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:20.921 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:20.921 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:20.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:20.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:20.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:21.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.092 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:21.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:21.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:03:21.193 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:21.223 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3736842097' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:21.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:21.362 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:21.463 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:21.498 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.499 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:21.600 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:21.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:21.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:21.772 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:21.859 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:21.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:21.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:21.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:21.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:22.010 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:22.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:22.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[108570]: pgmap v1337: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[108570]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[113926]: pgmap v1337: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:21 smithi042 ceph-mon[113926]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T15:03:22.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:22.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:21 smithi159 ceph-mon[102519]: pgmap v1337: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-13T15:03:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:21 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:21 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:21 smithi159 ceph-mon[102519]: osdmap e370: 8 total, 8 up, 8 in 2024-09-13T15:03:22.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:22.336 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:22.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:22.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:22.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.474 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:22.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:22.611 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:22.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:22.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:22.745 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:22.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:22.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:22.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:22.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:22.995 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:23.025 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:23.127 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:23.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:23.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:23.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:23.291 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:23.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:23.393 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:23.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:23.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:23.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.710 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.812 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:23.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:23.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:23.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:03:23.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:23.983 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:23.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:23.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:24.085 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:24.106 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:24.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:24.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:24.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:24.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:23 smithi159 ceph-mon[102519]: pgmap v1339: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:24.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:24.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:23 smithi042 ceph-mon[108570]: pgmap v1339: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:24.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:23 smithi042 ceph-mon[113926]: pgmap v1339: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:24.370 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:24.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:24.614 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:24.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:24.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:24.789 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:24.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:24.891 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:24.922 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:24.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:25.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.072 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:25.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:25.173 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:25.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:25.311 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:25.351 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:25.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:25.453 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:25.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:25.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:25.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:25.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:25.743 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:25.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:25.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:25.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:25.912 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:25.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:25.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:26.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.053 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:26.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:26.123 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:26.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.228 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:26.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:26.237 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:25 smithi042 ceph-mon[108570]: pgmap v1340: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:26.237 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:25 smithi042 ceph-mon[113926]: pgmap v1340: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:25 smithi159 ceph-mon[102519]: pgmap v1340: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-13T15:03:26.329 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:26.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.368 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:26.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:26.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.500 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:26.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:26.601 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:26.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:26.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:26.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:26.768 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:26.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:26.870 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:26.897 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:26.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:26.999 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:27.029 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:27.131 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:27.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:27.281 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:27.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:27.377 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:03:27.378 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:03:27.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:27.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:27.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:27.601 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:27.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:27.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:27.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:27.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:27.884 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:27.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:27.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:27.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:28.170 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:27 smithi159 ceph-mon[102519]: pgmap v1341: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-13T15:03:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:27 smithi159 ceph-mon[102519]: from='client.45950 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:28.272 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:28.278 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:27 smithi042 ceph-mon[108570]: pgmap v1341: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-13T15:03:28.278 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:27 smithi042 ceph-mon[108570]: from='client.45950 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:28.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:27 smithi042 ceph-mon[113926]: pgmap v1341: 99 pgs: 99 active+clean; 4.6 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-13T15:03:28.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:27 smithi042 ceph-mon[113926]: from='client.45950 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:03:28.310 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:28.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:28.412 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:28.466 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:28.568 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:28.604 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:28.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:28.706 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:28.745 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:28.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:28.799 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:03:28.800 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:28.829 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:03:28.829 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:28.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:28.851 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:03:28.852 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:28.868 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:03:28.869 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:28.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:28.870 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:28.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:28.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:29.001 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:03:29.001 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:29.036 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:03:29.036 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:29.072 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:03:29.073 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:29.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:29.120 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:03:29.121 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:29.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:29.194 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/247986670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:29.194 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/247986670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:29.234 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1341,"stamp":"2024-09-13T15:03:26.670325+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4948629501,"num_objects":75528,"num_object_clones":0,"num_object_copies":226584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75528,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171560,"num_write_kb":7903906,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":5182,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127394,"ondisk_log_size":127394,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22769760,"kb_used_data":14856664,"kb_used_omap":33,"kb_used_meta":7912862,"kb_avail":727191456,"statfs":{"total":767960285184,"available":744644050944,"internally_reserved":0,"allocated":15213223936,"data_stored":15197123130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33996,"internal_metadata":8102771508},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,2,8,5,0,3,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-599654400,"num_objects":-9150,"num_object_clones":0,"num_object_copies":-27450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9150,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006833"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":566,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931481+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:03:20.931481+0000","last_peered":"2024-09-13T15:03:20.931481+0000","last_clean":"2024-09-13T15:03:20.931481+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:03:20.931481+0000","last_undegraded":"2024-09-13T15:03:20.931481+0000","last_fullsized":"2024-09-13T15:03:20.931481+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":62,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878219+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:03:03.878219+0000","last_peered":"2024-09-13T15:03:03.878219+0000","last_clean":"2024-09-13T15:03:03.878219+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:03:03.878219+0000","last_undegraded":"2024-09-13T15:03:03.878219+0000","last_fullsized":"2024-09-13T15:03:03.878219+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":130,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931818+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:03:20.931818+0000","last_peered":"2024-09-13T15:03:20.931818+0000","last_clean":"2024-09-13T15:03:20.931818+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:03:20.931818+0000","last_undegraded":"2024-09-13T15:03:20.931818+0000","last_fullsized":"2024-09-13T15:03:20.931818+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924852+0000","last_change":"2024-09-13T15:00:58.195537+0000","last_active":"2024-09-13T15:03:20.924852+0000","last_peered":"2024-09-13T15:03:20.924852+0000","last_clean":"2024-09-13T15:03:20.924852+0000","last_became_active":"2024-09-13T15:00:58.194028+0000","last_became_peered":"2024-09-13T15:00:58.194028+0000","last_unstale":"2024-09-13T15:03:20.924852+0000","last_undegraded":"2024-09-13T15:03:20.924852+0000","last_fullsized":"2024-09-13T15:03:20.924852+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:47.298186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924320+0000","last_change":"2024-09-13T15:00:58.197669+0000","last_active":"2024-09-13T15:03:20.924320+0000","last_peered":"2024-09-13T15:03:20.924320+0000","last_clean":"2024-09-13T15:03:20.924320+0000","last_became_active":"2024-09-13T15:00:58.196512+0000","last_became_peered":"2024-09-13T15:00:58.196512+0000","last_unstale":"2024-09-13T15:03:20.924320+0000","last_undegraded":"2024-09-13T15:03:20.924320+0000","last_fullsized":"2024-09-13T15:03:20.924320+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:23:15.533805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878092+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:03:03.878092+0000","last_peered":"2024-09-13T15:03:03.878092+0000","last_clean":"2024-09-13T15:03:03.878092+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:03:03.878092+0000","last_undegraded":"2024-09-13T15:03:03.878092+0000","last_fullsized":"2024-09-13T15:03:03.878092+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":592,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.932120+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:03:20.932120+0000","last_peered":"2024-09-13T15:03:20.932120+0000","last_clean":"2024-09-13T15:03:20.932120+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:03:20.932120+0000","last_undegraded":"2024-09-13T15:03:20.932120+0000","last_fullsized":"2024-09-13T15:03:20.932120+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":443,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876148+0000","last_change":"2024-09-13T15:00:58.197147+0000","last_active":"2024-09-13T15:03:03.876148+0000","last_peered":"2024-09-13T15:03:03.876148+0000","last_clean":"2024-09-13T15:03:03.876148+0000","last_became_active":"2024-09-13T15:00:58.196810+0000","last_became_peered":"2024-09-13T15:00:58.196810+0000","last_unstale":"2024-09-13T15:03:03.876148+0000","last_undegraded":"2024-09-13T15:03:03.876148+0000","last_fullsized":"2024-09-13T15:03:03.876148+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:27:03.062492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.1","version":"370'10784","reported_seq":10820,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.124682+0000","last_change":"2024-09-13T15:00:51.158623+0000","last_active":"2024-09-13T15:03:26.124682+0000","last_peered":"2024-09-13T15:03:26.124682+0000","last_clean":"2024-09-13T15:03:26.124682+0000","last_became_active":"2024-09-13T15:00:51.158283+0000","last_became_peered":"2024-09-13T15:00:51.158283+0000","last_unstale":"2024-09-13T15:03:26.124682+0000","last_undegraded":"2024-09-13T15:03:26.124682+0000","last_fullsized":"2024-09-13T15:03:26.124682+0000","mapping_epoch":344,"log_start":"352'2600","ondisk_log_start":"352'2600","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8184,"log_dups_size":0,"ondisk_log_size":8184,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:37:32.843875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309592064,"num_objects":4724,"num_object_clones":0,"num_object_copies":14172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10784,"num_write_kb":496256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":71,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877865+0000","last_change":"2024-09-13T15:00:59.188914+0000","last_active":"2024-09-13T15:03:03.877865+0000","last_peered":"2024-09-13T15:03:03.877865+0000","last_clean":"2024-09-13T15:03:03.877865+0000","last_became_active":"2024-09-13T15:00:59.188606+0000","last_became_peered":"2024-09-13T15:00:59.188606+0000","last_unstale":"2024-09-13T15:03:03.877865+0000","last_undegraded":"2024-09-13T15:03:03.877865+0000","last_fullsized":"2024-09-13T15:03:03.877865+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:51:32.701420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.875415+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:03:03.875415+0000","last_peered":"2024-09-13T15:03:03.875415+0000","last_clean":"2024-09-13T15:03:03.875415+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:03:03.875415+0000","last_undegraded":"2024-09-13T15:03:03.875415+0000","last_fullsized":"2024-09-13T15:03:03.875415+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.0","version":"370'10691","reported_seq":10755,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:24.928620+0000","last_change":"2024-09-13T15:00:51.638677+0000","last_active":"2024-09-13T15:03:24.928620+0000","last_peered":"2024-09-13T15:03:24.928620+0000","last_clean":"2024-09-13T15:03:24.928620+0000","last_became_active":"2024-09-13T15:00:51.638338+0000","last_became_peered":"2024-09-13T15:00:51.638338+0000","last_unstale":"2024-09-13T15:03:24.928620+0000","last_undegraded":"2024-09-13T15:03:24.928620+0000","last_fullsized":"2024-09-13T15:03:24.928620+0000","mapping_epoch":344,"log_start":"352'2500","ondisk_log_start":"352'2500","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:01:43.270299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305070080,"num_objects":4655,"num_object_clones":0,"num_object_copies":13965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10691,"num_write_kb":491072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":60,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883802+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T15:03:03.883802+0000","last_peered":"2024-09-13T15:03:03.883802+0000","last_clean":"2024-09-13T15:03:03.883802+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T15:03:03.883802+0000","last_undegraded":"2024-09-13T15:03:03.883802+0000","last_fullsized":"2024-09-13T15:03:03.883802+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:20.897906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878276+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T15:03:03.878276+0000","last_peered":"2024-09-13T15:03:03.878276+0000","last_clean":"2024-09-13T15:03:03.878276+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T15:03:03.878276+0000","last_undegraded":"2024-09-13T15:03:03.878276+0000","last_fullsized":"2024-09-13T15:03:03.878276+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.3","version":"370'10751","reported_seq":10807,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.279932+0000","last_change":"2024-09-13T15:00:58.195408+0000","last_active":"2024-09-13T15:03:26.279932+0000","last_peered":"2024-09-13T15:03:26.279932+0000","last_clean":"2024-09-13T15:03:26.279932+0000","last_became_active":"2024-09-13T15:00:58.195010+0000","last_became_peered":"2024-09-13T15:00:58.195010+0000","last_unstale":"2024-09-13T15:03:26.279932+0000","last_undegraded":"2024-09-13T15:03:26.279932+0000","last_fullsized":"2024-09-13T15:03:26.279932+0000","mapping_epoch":348,"log_start":"352'2100","ondisk_log_start":"352'2100","created":344,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8651,"log_dups_size":0,"ondisk_log_size":8651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:38:41.965793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307822592,"num_objects":4697,"num_object_clones":0,"num_object_copies":14091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10751,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":39,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.874189+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:03:03.874189+0000","last_peered":"2024-09-13T15:03:03.874189+0000","last_clean":"2024-09-13T15:03:03.874189+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:03:03.874189+0000","last_undegraded":"2024-09-13T15:03:03.874189+0000","last_fullsized":"2024-09-13T15:03:03.874189+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":344,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.875642+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T15:03:03.875642+0000","last_peered":"2024-09-13T15:03:03.875642+0000","last_clean":"2024-09-13T15:03:03.875642+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T15:03:03.875642+0000","last_undegraded":"2024-09-13T15:03:03.875642+0000","last_fullsized":"2024-09-13T15:03:03.875642+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.2","version":"370'10522","reported_seq":10578,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:22.185363+0000","last_change":"2024-09-13T15:01:33.864621+0000","last_active":"2024-09-13T15:03:22.185363+0000","last_peered":"2024-09-13T15:03:22.185363+0000","last_clean":"2024-09-13T15:03:22.185363+0000","last_became_active":"2024-09-13T15:01:33.856103+0000","last_became_peered":"2024-09-13T15:01:33.856103+0000","last_unstale":"2024-09-13T15:03:22.185363+0000","last_undegraded":"2024-09-13T15:03:22.185363+0000","last_fullsized":"2024-09-13T15:03:22.185363+0000","mapping_epoch":354,"log_start":"352'2800","ondisk_log_start":"352'2800","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":7722,"log_dups_size":0,"ondisk_log_size":7722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:04:34.307026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323092480,"num_objects":4930,"num_object_clones":0,"num_object_copies":14790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10522,"num_write_kb":494464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931601+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:03:20.931601+0000","last_peered":"2024-09-13T15:03:20.931601+0000","last_clean":"2024-09-13T15:03:20.931601+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:03:20.931601+0000","last_undegraded":"2024-09-13T15:03:20.931601+0000","last_fullsized":"2024-09-13T15:03:20.931601+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":202,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.925063+0000","last_change":"2024-09-13T15:00:59.190451+0000","last_active":"2024-09-13T15:03:20.925063+0000","last_peered":"2024-09-13T15:03:20.925063+0000","last_clean":"2024-09-13T15:03:20.925063+0000","last_became_active":"2024-09-13T15:00:59.189511+0000","last_became_peered":"2024-09-13T15:00:59.189511+0000","last_unstale":"2024-09-13T15:03:20.925063+0000","last_undegraded":"2024-09-13T15:03:20.925063+0000","last_fullsized":"2024-09-13T15:03:20.925063+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:24:58.355252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.5","version":"370'10774","reported_seq":10816,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.223890+0000","last_change":"2024-09-13T15:00:58.196823+0000","last_active":"2024-09-13T15:03:26.223890+0000","last_peered":"2024-09-13T15:03:26.223890+0000","last_clean":"2024-09-13T15:03:26.223890+0000","last_became_active":"2024-09-13T15:00:58.196522+0000","last_became_peered":"2024-09-13T15:00:58.196522+0000","last_unstale":"2024-09-13T15:03:26.223890+0000","last_undegraded":"2024-09-13T15:03:26.223890+0000","last_fullsized":"2024-09-13T15:03:26.223890+0000","mapping_epoch":348,"log_start":"355'4100","ondisk_log_start":"355'4100","created":344,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6674,"log_dups_size":0,"ondisk_log_size":6674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:22:09.740245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303955968,"num_objects":4638,"num_object_clones":0,"num_object_copies":13914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10774,"num_write_kb":493184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":918,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927973+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:03:20.927973+0000","last_peered":"2024-09-13T15:03:20.927973+0000","last_clean":"2024-09-13T15:03:20.927973+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:03:20.927973+0000","last_undegraded":"2024-09-13T15:03:20.927973+0000","last_fullsized":"2024-09-13T15:03:20.927973+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":630,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.933594+0000","last_change":"2024-09-13T15:00:59.189216+0000","last_active":"2024-09-13T15:03:20.933594+0000","last_peered":"2024-09-13T15:03:20.933594+0000","last_clean":"2024-09-13T15:03:20.933594+0000","last_became_active":"2024-09-13T15:00:59.188828+0000","last_became_peered":"2024-09-13T15:00:59.188828+0000","last_unstale":"2024-09-13T15:03:20.933594+0000","last_undegraded":"2024-09-13T15:03:20.933594+0000","last_fullsized":"2024-09-13T15:03:20.933594+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:02.854311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877938+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:03:03.877938+0000","last_peered":"2024-09-13T15:03:03.877938+0000","last_clean":"2024-09-13T15:03:03.877938+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:03:03.877938+0000","last_undegraded":"2024-09-13T15:03:03.877938+0000","last_fullsized":"2024-09-13T15:03:03.877938+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.4","version":"370'10570","reported_seq":10606,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.109539+0000","last_change":"2024-09-13T15:00:51.158719+0000","last_active":"2024-09-13T15:03:26.109539+0000","last_peered":"2024-09-13T15:03:26.109539+0000","last_clean":"2024-09-13T15:03:26.109539+0000","last_became_active":"2024-09-13T15:00:51.158433+0000","last_became_peered":"2024-09-13T15:00:51.158433+0000","last_unstale":"2024-09-13T15:03:26.109539+0000","last_undegraded":"2024-09-13T15:03:26.109539+0000","last_fullsized":"2024-09-13T15:03:26.109539+0000","mapping_epoch":344,"log_start":"352'2400","ondisk_log_start":"352'2400","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8170,"log_dups_size":0,"ondisk_log_size":8170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:12:33.014048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298319872,"num_objects":4552,"num_object_clones":0,"num_object_copies":13656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10570,"num_write_kb":483904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":415,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.919550+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:03:20.919550+0000","last_peered":"2024-09-13T15:03:20.919550+0000","last_clean":"2024-09-13T15:03:20.919550+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:03:20.919550+0000","last_undegraded":"2024-09-13T15:03:20.919550+0000","last_fullsized":"2024-09-13T15:03:20.919550+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":392,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.875779+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T15:03:03.875779+0000","last_peered":"2024-09-13T15:03:03.875779+0000","last_clean":"2024-09-13T15:03:03.875779+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T15:03:03.875779+0000","last_undegraded":"2024-09-13T15:03:03.875779+0000","last_fullsized":"2024-09-13T15:03:03.875779+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880068+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T15:03:03.880068+0000","last_peered":"2024-09-13T15:03:03.880068+0000","last_clean":"2024-09-13T15:03:03.880068+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T15:03:03.880068+0000","last_undegraded":"2024-09-13T15:03:03.880068+0000","last_fullsized":"2024-09-13T15:03:03.880068+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.7","version":"370'10615","reported_seq":10650,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.158884+0000","last_change":"2024-09-13T15:00:51.641016+0000","last_active":"2024-09-13T15:03:26.158884+0000","last_peered":"2024-09-13T15:03:26.158884+0000","last_clean":"2024-09-13T15:03:26.158884+0000","last_became_active":"2024-09-13T15:00:51.640705+0000","last_became_peered":"2024-09-13T15:00:51.640705+0000","last_unstale":"2024-09-13T15:03:26.158884+0000","last_undegraded":"2024-09-13T15:03:26.158884+0000","last_fullsized":"2024-09-13T15:03:26.158884+0000","mapping_epoch":344,"log_start":"352'2500","ondisk_log_start":"352'2500","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:33:37.313904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294846464,"num_objects":4499,"num_object_clones":0,"num_object_copies":13497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10615,"num_write_kb":483648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878161+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:03:03.878161+0000","last_peered":"2024-09-13T15:03:03.878161+0000","last_clean":"2024-09-13T15:03:03.878161+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:03:03.878161+0000","last_undegraded":"2024-09-13T15:03:03.878161+0000","last_fullsized":"2024-09-13T15:03:03.878161+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.883878+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T15:03:03.883878+0000","last_peered":"2024-09-13T15:03:03.883878+0000","last_clean":"2024-09-13T15:03:03.883878+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T15:03:03.883878+0000","last_undegraded":"2024-09-13T15:03:03.883878+0000","last_fullsized":"2024-09-13T15:03:03.883878+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"13.6","version":"370'10880","reported_seq":10916,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.229538+0000","last_change":"2024-09-13T15:00:51.159137+0000","last_active":"2024-09-13T15:03:26.229538+0000","last_peered":"2024-09-13T15:03:26.229538+0000","last_clean":"2024-09-13T15:03:26.229538+0000","last_became_active":"2024-09-13T15:00:51.158825+0000","last_became_peered":"2024-09-13T15:00:51.158825+0000","last_unstale":"2024-09-13T15:03:26.229538+0000","last_undegraded":"2024-09-13T15:03:26.229538+0000","last_fullsized":"2024-09-13T15:03:26.229538+0000","mapping_epoch":344,"log_start":"355'4200","ondisk_log_start":"355'4200","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:36.728759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312606720,"num_objects":4770,"num_object_clones":0,"num_object_copies":14310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10880,"num_write_kb":500800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":415,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.917796+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:03:20.917796+0000","last_peered":"2024-09-13T15:03:20.917796+0000","last_clean":"2024-09-13T15:03:20.917796+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:03:20.917796+0000","last_undegraded":"2024-09-13T15:03:20.917796+0000","last_fullsized":"2024-09-13T15:03:20.917796+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":569,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927879+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:03:20.927879+0000","last_peered":"2024-09-13T15:03:20.927879+0000","last_clean":"2024-09-13T15:03:20.927879+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:03:20.927879+0000","last_undegraded":"2024-09-13T15:03:20.927879+0000","last_fullsized":"2024-09-13T15:03:20.927879+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":582,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916466+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:03:20.916466+0000","last_peered":"2024-09-13T15:03:20.916466+0000","last_clean":"2024-09-13T15:03:20.916466+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:03:20.916466+0000","last_undegraded":"2024-09-13T15:03:20.916466+0000","last_fullsized":"2024-09-13T15:03:20.916466+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.c","version":"370'10645","reported_seq":10709,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:24.882268+0000","last_change":"2024-09-13T15:00:51.159165+0000","last_active":"2024-09-13T15:03:24.882268+0000","last_peered":"2024-09-13T15:03:24.882268+0000","last_clean":"2024-09-13T15:03:24.882268+0000","last_became_active":"2024-09-13T15:00:51.158866+0000","last_became_peered":"2024-09-13T15:00:51.158866+0000","last_unstale":"2024-09-13T15:03:24.882268+0000","last_undegraded":"2024-09-13T15:03:24.882268+0000","last_fullsized":"2024-09-13T15:03:24.882268+0000","mapping_epoch":344,"log_start":"352'2500","ondisk_log_start":"352'2500","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:41:18.372131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305332224,"num_objects":4659,"num_object_clones":0,"num_object_copies":13977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10645,"num_write_kb":489728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":538,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.923809+0000","last_change":"2024-09-13T15:00:58.193515+0000","last_active":"2024-09-13T15:03:20.923809+0000","last_peered":"2024-09-13T15:03:20.923809+0000","last_clean":"2024-09-13T15:03:20.923809+0000","last_became_active":"2024-09-13T15:00:58.193108+0000","last_became_peered":"2024-09-13T15:00:58.193108+0000","last_unstale":"2024-09-13T15:03:20.923809+0000","last_undegraded":"2024-09-13T15:03:20.923809+0000","last_fullsized":"2024-09-13T15:03:20.923809+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:33:30.120095+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":53,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.918300+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:03:20.918300+0000","last_peered":"2024-09-13T15:03:20.918300+0000","last_clean":"2024-09-13T15:03:20.918300+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:03:20.918300+0000","last_undegraded":"2024-09-13T15:03:20.918300+0000","last_fullsized":"2024-09-13T15:03:20.918300+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":554,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927578+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:03:20.927578+0000","last_peered":"2024-09-13T15:03:20.927578+0000","last_clean":"2024-09-13T15:03:20.927578+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:03:20.927578+0000","last_undegraded":"2024-09-13T15:03:20.927578+0000","last_fullsized":"2024-09-13T15:03:20.927578+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":354,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.874262+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:03:03.874262+0000","last_peered":"2024-09-13T15:03:03.874262+0000","last_clean":"2024-09-13T15:03:03.874262+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:03:03.874262+0000","last_undegraded":"2024-09-13T15:03:03.874262+0000","last_fullsized":"2024-09-13T15:03:03.874262+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"13.f","version":"370'10729","reported_seq":10765,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.143556+0000","last_change":"2024-09-13T15:00:51.640917+0000","last_active":"2024-09-13T15:03:26.143556+0000","last_peered":"2024-09-13T15:03:26.143556+0000","last_clean":"2024-09-13T15:03:26.143556+0000","last_became_active":"2024-09-13T15:00:51.640598+0000","last_became_peered":"2024-09-13T15:00:51.640598+0000","last_unstale":"2024-09-13T15:03:26.143556+0000","last_undegraded":"2024-09-13T15:03:26.143556+0000","last_fullsized":"2024-09-13T15:03:26.143556+0000","mapping_epoch":344,"log_start":"352'2600","ondisk_log_start":"352'2600","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:37:45.675640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10729,"num_write_kb":491776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":849,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.915763+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:03:20.915763+0000","last_peered":"2024-09-13T15:03:20.915763+0000","last_clean":"2024-09-13T15:03:20.915763+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:03:20.915763+0000","last_undegraded":"2024-09-13T15:03:20.915763+0000","last_fullsized":"2024-09-13T15:03:20.915763+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":566,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.915804+0000","last_change":"2024-09-13T15:00:58.198644+0000","last_active":"2024-09-13T15:03:20.915804+0000","last_peered":"2024-09-13T15:03:20.915804+0000","last_clean":"2024-09-13T15:03:20.915804+0000","last_became_active":"2024-09-13T15:00:58.198334+0000","last_became_peered":"2024-09-13T15:00:58.198334+0000","last_unstale":"2024-09-13T15:03:20.915804+0000","last_undegraded":"2024-09-13T15:03:20.915804+0000","last_fullsized":"2024-09-13T15:03:20.915804+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:39:52.461192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":203,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931114+0000","last_change":"2024-09-13T15:00:59.188848+0000","last_active":"2024-09-13T15:03:20.931114+0000","last_peered":"2024-09-13T15:03:20.931114+0000","last_clean":"2024-09-13T15:03:20.931114+0000","last_became_active":"2024-09-13T15:00:59.188521+0000","last_became_peered":"2024-09-13T15:00:59.188521+0000","last_unstale":"2024-09-13T15:03:20.931114+0000","last_undegraded":"2024-09-13T15:03:20.931114+0000","last_fullsized":"2024-09-13T15:03:20.931114+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:47:45.161336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924493+0000","last_change":"2024-09-13T15:00:58.196782+0000","last_active":"2024-09-13T15:03:20.924493+0000","last_peered":"2024-09-13T15:03:20.924493+0000","last_clean":"2024-09-13T15:03:20.924493+0000","last_became_active":"2024-09-13T15:00:58.194669+0000","last_became_peered":"2024-09-13T15:00:58.194669+0000","last_unstale":"2024-09-13T15:03:20.924493+0000","last_undegraded":"2024-09-13T15:03:20.924493+0000","last_fullsized":"2024-09-13T15:03:20.924493+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:55:56.299128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.e","version":"370'10782","reported_seq":10824,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.221462+0000","last_change":"2024-09-13T15:01:33.862631+0000","last_active":"2024-09-13T15:03:26.221462+0000","last_peered":"2024-09-13T15:03:26.221462+0000","last_clean":"2024-09-13T15:03:26.221462+0000","last_became_active":"2024-09-13T15:01:33.855592+0000","last_became_peered":"2024-09-13T15:01:33.855592+0000","last_unstale":"2024-09-13T15:03:26.221462+0000","last_undegraded":"2024-09-13T15:03:26.221462+0000","last_fullsized":"2024-09-13T15:03:26.221462+0000","mapping_epoch":354,"log_start":"355'4100","ondisk_log_start":"355'4100","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6682,"log_dups_size":0,"ondisk_log_size":6682,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:33:37.590412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10782,"num_write_kb":492864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":613,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924771+0000","last_change":"2024-09-13T15:00:59.187059+0000","last_active":"2024-09-13T15:03:20.924771+0000","last_peered":"2024-09-13T15:03:20.924771+0000","last_clean":"2024-09-13T15:03:20.924771+0000","last_became_active":"2024-09-13T15:00:59.186471+0000","last_became_peered":"2024-09-13T15:00:59.186471+0000","last_unstale":"2024-09-13T15:03:20.924771+0000","last_undegraded":"2024-09-13T15:03:20.924771+0000","last_fullsized":"2024-09-13T15:03:20.924771+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:56:15.251496+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":618,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.919139+0000","last_change":"2024-09-13T15:00:58.191231+0000","last_active":"2024-09-13T15:03:20.919139+0000","last_peered":"2024-09-13T15:03:20.919139+0000","last_clean":"2024-09-13T15:03:20.919139+0000","last_became_active":"2024-09-13T15:00:58.190874+0000","last_became_peered":"2024-09-13T15:00:58.190874+0000","last_unstale":"2024-09-13T15:03:20.919139+0000","last_undegraded":"2024-09-13T15:03:20.919139+0000","last_fullsized":"2024-09-13T15:03:20.919139+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:19:42.995687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":567,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.915997+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:03:20.915997+0000","last_peered":"2024-09-13T15:03:20.915997+0000","last_clean":"2024-09-13T15:03:20.915997+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:03:20.915997+0000","last_undegraded":"2024-09-13T15:03:20.915997+0000","last_fullsized":"2024-09-13T15:03:20.915997+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":628,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916320+0000","last_change":"2024-09-13T15:00:58.189504+0000","last_active":"2024-09-13T15:03:20.916320+0000","last_peered":"2024-09-13T15:03:20.916320+0000","last_clean":"2024-09-13T15:03:20.916320+0000","last_became_active":"2024-09-13T15:00:58.189158+0000","last_became_peered":"2024-09-13T15:00:58.189158+0000","last_unstale":"2024-09-13T15:03:20.916320+0000","last_undegraded":"2024-09-13T15:03:20.916320+0000","last_fullsized":"2024-09-13T15:03:20.916320+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:23.156063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.d","version":"370'10888","reported_seq":10925,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.171953+0000","last_change":"2024-09-13T15:00:51.638684+0000","last_active":"2024-09-13T15:03:26.171953+0000","last_peered":"2024-09-13T15:03:26.171953+0000","last_clean":"2024-09-13T15:03:26.171953+0000","last_became_active":"2024-09-13T15:00:51.638349+0000","last_became_peered":"2024-09-13T15:00:51.638349+0000","last_unstale":"2024-09-13T15:03:26.171953+0000","last_undegraded":"2024-09-13T15:03:26.171953+0000","last_fullsized":"2024-09-13T15:03:26.171953+0000","mapping_epoch":344,"log_start":"355'4200","ondisk_log_start":"355'4200","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":6688,"log_dups_size":0,"ondisk_log_size":6688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:53:23.643248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939032,"num_objects":4654,"num_object_clones":0,"num_object_copies":13962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4654,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10888,"num_write_kb":497281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":549,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931225+0000","last_change":"2024-09-13T15:01:33.848879+0000","last_active":"2024-09-13T15:03:20.931225+0000","last_peered":"2024-09-13T15:03:20.931225+0000","last_clean":"2024-09-13T15:03:20.931225+0000","last_became_active":"2024-09-13T15:01:33.848475+0000","last_became_peered":"2024-09-13T15:01:33.848475+0000","last_unstale":"2024-09-13T15:03:20.931225+0000","last_undegraded":"2024-09-13T15:03:20.931225+0000","last_fullsized":"2024-09-13T15:03:20.931225+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.295859+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:40:46.934186+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":58,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.917178+0000","last_change":"2024-09-13T15:01:33.852433+0000","last_active":"2024-09-13T15:03:20.917178+0000","last_peered":"2024-09-13T15:03:20.917178+0000","last_clean":"2024-09-13T15:03:20.917178+0000","last_became_active":"2024-09-13T15:01:33.852015+0000","last_became_peered":"2024-09-13T15:01:33.852015+0000","last_unstale":"2024-09-13T15:03:20.917178+0000","last_undegraded":"2024-09-13T15:03:20.917178+0000","last_fullsized":"2024-09-13T15:03:20.917178+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:52:03.246939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.918533+0000","last_change":"2024-09-13T15:00:59.305010+0000","last_active":"2024-09-13T15:03:20.918533+0000","last_peered":"2024-09-13T15:03:20.918533+0000","last_clean":"2024-09-13T15:03:20.918533+0000","last_became_active":"2024-09-13T15:00:59.304744+0000","last_became_peered":"2024-09-13T15:00:59.304744+0000","last_unstale":"2024-09-13T15:03:20.918533+0000","last_undegraded":"2024-09-13T15:03:20.918533+0000","last_fullsized":"2024-09-13T15:03:20.918533+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:58:57.917883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":580,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916110+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:03:20.916110+0000","last_peered":"2024-09-13T15:03:20.916110+0000","last_clean":"2024-09-13T15:03:20.916110+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:03:20.916110+0000","last_undegraded":"2024-09-13T15:03:20.916110+0000","last_fullsized":"2024-09-13T15:03:20.916110+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.b","version":"370'10561","reported_seq":10619,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:22.173038+0000","last_change":"2024-09-13T15:01:33.863073+0000","last_active":"2024-09-13T15:03:22.173038+0000","last_peered":"2024-09-13T15:03:22.173038+0000","last_clean":"2024-09-13T15:03:22.173038+0000","last_became_active":"2024-09-13T15:01:33.855365+0000","last_became_peered":"2024-09-13T15:01:33.855365+0000","last_unstale":"2024-09-13T15:03:22.173038+0000","last_undegraded":"2024-09-13T15:03:22.173038+0000","last_fullsized":"2024-09-13T15:03:22.173038+0000","mapping_epoch":354,"log_start":"352'2800","ondisk_log_start":"352'2800","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":7761,"log_dups_size":0,"ondisk_log_size":7761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:19:18.315313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323158016,"num_objects":4931,"num_object_clones":0,"num_object_copies":14793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10561,"num_write_kb":495744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":621,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876595+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:03:03.876595+0000","last_peered":"2024-09-13T15:03:03.876595+0000","last_clean":"2024-09-13T15:03:03.876595+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:03:03.876595+0000","last_undegraded":"2024-09-13T15:03:03.876595+0000","last_fullsized":"2024-09-13T15:03:03.876595+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":147,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931959+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:03:20.931959+0000","last_peered":"2024-09-13T15:03:20.931959+0000","last_clean":"2024-09-13T15:03:20.931959+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:03:20.931959+0000","last_undegraded":"2024-09-13T15:03:20.931959+0000","last_fullsized":"2024-09-13T15:03:20.931959+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":643,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916292+0000","last_change":"2024-09-13T15:00:58.197235+0000","last_active":"2024-09-13T15:03:20.916292+0000","last_peered":"2024-09-13T15:03:20.916292+0000","last_clean":"2024-09-13T15:03:20.916292+0000","last_became_active":"2024-09-13T15:00:58.196938+0000","last_became_peered":"2024-09-13T15:00:58.196938+0000","last_unstale":"2024-09-13T15:03:20.916292+0000","last_undegraded":"2024-09-13T15:03:20.916292+0000","last_fullsized":"2024-09-13T15:03:20.916292+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:33:33.195527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":506,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.917047+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:03:20.917047+0000","last_peered":"2024-09-13T15:03:20.917047+0000","last_clean":"2024-09-13T15:03:20.917047+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:03:20.917047+0000","last_undegraded":"2024-09-13T15:03:20.917047+0000","last_fullsized":"2024-09-13T15:03:20.917047+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"13.a","version":"370'10632","reported_seq":10696,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:22.644665+0000","last_change":"2024-09-13T15:00:51.158952+0000","last_active":"2024-09-13T15:03:22.644665+0000","last_peered":"2024-09-13T15:03:22.644665+0000","last_clean":"2024-09-13T15:03:22.644665+0000","last_became_active":"2024-09-13T15:00:51.158650+0000","last_became_peered":"2024-09-13T15:00:51.158650+0000","last_unstale":"2024-09-13T15:03:22.644665+0000","last_undegraded":"2024-09-13T15:03:22.644665+0000","last_fullsized":"2024-09-13T15:03:22.644665+0000","mapping_epoch":344,"log_start":"352'2000","ondisk_log_start":"352'2000","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8632,"log_dups_size":0,"ondisk_log_size":8632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:05:58.684962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316932096,"num_objects":4836,"num_object_clones":0,"num_object_copies":14508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10632,"num_write_kb":494976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":670,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.874224+0000","last_change":"2024-09-13T15:00:58.199832+0000","last_active":"2024-09-13T15:03:03.874224+0000","last_peered":"2024-09-13T15:03:03.874224+0000","last_clean":"2024-09-13T15:03:03.874224+0000","last_became_active":"2024-09-13T15:00:58.199563+0000","last_became_peered":"2024-09-13T15:00:58.199563+0000","last_unstale":"2024-09-13T15:03:03.874224+0000","last_undegraded":"2024-09-13T15:03:03.874224+0000","last_fullsized":"2024-09-13T15:03:03.874224+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:27:58.757475+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":567,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916201+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:03:20.916201+0000","last_peered":"2024-09-13T15:03:20.916201+0000","last_clean":"2024-09-13T15:03:20.916201+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:03:20.916201+0000","last_undegraded":"2024-09-13T15:03:20.916201+0000","last_fullsized":"2024-09-13T15:03:20.916201+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":203,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924135+0000","last_change":"2024-09-13T15:00:59.185867+0000","last_active":"2024-09-13T15:03:20.924135+0000","last_peered":"2024-09-13T15:03:20.924135+0000","last_clean":"2024-09-13T15:03:20.924135+0000","last_became_active":"2024-09-13T15:00:59.184841+0000","last_became_peered":"2024-09-13T15:00:59.184841+0000","last_unstale":"2024-09-13T15:03:20.924135+0000","last_undegraded":"2024-09-13T15:03:20.924135+0000","last_fullsized":"2024-09-13T15:03:20.924135+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:58.991299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.925261+0000","last_change":"2024-09-13T15:00:58.192299+0000","last_active":"2024-09-13T15:03:20.925261+0000","last_peered":"2024-09-13T15:03:20.925261+0000","last_clean":"2024-09-13T15:03:20.925261+0000","last_became_active":"2024-09-13T15:00:58.191940+0000","last_became_peered":"2024-09-13T15:00:58.191940+0000","last_unstale":"2024-09-13T15:03:20.925261+0000","last_undegraded":"2024-09-13T15:03:20.925261+0000","last_fullsized":"2024-09-13T15:03:20.925261+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:30:31.247758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"13.9","version":"370'10861","reported_seq":10926,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:26.230471+0000","last_change":"2024-09-13T15:01:33.853395+0000","last_active":"2024-09-13T15:03:26.230471+0000","last_peered":"2024-09-13T15:03:26.230471+0000","last_clean":"2024-09-13T15:03:26.230471+0000","last_became_active":"2024-09-13T15:01:33.849065+0000","last_became_peered":"2024-09-13T15:01:33.849065+0000","last_unstale":"2024-09-13T15:03:26.230471+0000","last_undegraded":"2024-09-13T15:03:26.230471+0000","last_fullsized":"2024-09-13T15:03:26.230471+0000","mapping_epoch":354,"log_start":"352'2200","ondisk_log_start":"352'2200","created":344,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8661,"log_dups_size":0,"ondisk_log_size":8661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:38:01.268141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309526528,"num_objects":4723,"num_object_clones":0,"num_object_copies":14169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10861,"num_write_kb":498688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":550,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.919103+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:03:20.919103+0000","last_peered":"2024-09-13T15:03:20.919103+0000","last_clean":"2024-09-13T15:03:20.919103+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:03:20.919103+0000","last_undegraded":"2024-09-13T15:03:20.919103+0000","last_fullsized":"2024-09-13T15:03:20.919103+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":588,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.932309+0000","last_change":"2024-09-13T15:00:58.198098+0000","last_active":"2024-09-13T15:03:20.932309+0000","last_peered":"2024-09-13T15:03:20.932309+0000","last_clean":"2024-09-13T15:03:20.932309+0000","last_became_active":"2024-09-13T15:00:58.197740+0000","last_became_peered":"2024-09-13T15:00:58.197740+0000","last_unstale":"2024-09-13T15:03:20.932309+0000","last_undegraded":"2024-09-13T15:03:20.932309+0000","last_fullsized":"2024-09-13T15:03:20.932309+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:45:20.126199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":581,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916345+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:03:20.916345+0000","last_peered":"2024-09-13T15:03:20.916345+0000","last_clean":"2024-09-13T15:03:20.916345+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:03:20.916345+0000","last_undegraded":"2024-09-13T15:03:20.916345+0000","last_fullsized":"2024-09-13T15:03:20.916345+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":592,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927690+0000","last_change":"2024-09-13T15:00:58.198790+0000","last_active":"2024-09-13T15:03:20.927690+0000","last_peered":"2024-09-13T15:03:20.927690+0000","last_clean":"2024-09-13T15:03:20.927690+0000","last_became_active":"2024-09-13T15:00:58.198508+0000","last_became_peered":"2024-09-13T15:00:58.198508+0000","last_unstale":"2024-09-13T15:03:20.927690+0000","last_undegraded":"2024-09-13T15:03:20.927690+0000","last_fullsized":"2024-09-13T15:03:20.927690+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:50:51.003072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"13.8","version":"370'10811","reported_seq":10875,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:22.630741+0000","last_change":"2024-09-13T15:00:51.157239+0000","last_active":"2024-09-13T15:03:22.630741+0000","last_peered":"2024-09-13T15:03:22.630741+0000","last_clean":"2024-09-13T15:03:22.630741+0000","last_became_active":"2024-09-13T15:00:51.156946+0000","last_became_peered":"2024-09-13T15:00:51.156946+0000","last_unstale":"2024-09-13T15:03:22.630741+0000","last_undegraded":"2024-09-13T15:03:22.630741+0000","last_fullsized":"2024-09-13T15:03:22.630741+0000","mapping_epoch":344,"log_start":"352'2200","ondisk_log_start":"352'2200","created":344,"last_epoch_clean":345,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:00:50.143914+0000","last_clean_scrub_stamp":"2024-09-13T15:00:50.143914+0000","objects_scrubbed":0,"log_size":8611,"log_dups_size":0,"ondisk_log_size":8611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:07:54.326025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326041600,"num_objects":4975,"num_object_clones":0,"num_object_copies":14925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10811,"num_write_kb":505152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":828,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931335+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:03:20.931335+0000","last_peered":"2024-09-13T15:03:20.931335+0000","last_clean":"2024-09-13T15:03:20.931335+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:03:20.931335+0000","last_undegraded":"2024-09-13T15:03:20.931335+0000","last_fullsized":"2024-09-13T15:03:20.931335+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":69,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924191+0000","last_change":"2024-09-13T15:00:59.187967+0000","last_active":"2024-09-13T15:03:20.924191+0000","last_peered":"2024-09-13T15:03:20.924191+0000","last_clean":"2024-09-13T15:03:20.924191+0000","last_became_active":"2024-09-13T15:00:59.187518+0000","last_became_peered":"2024-09-13T15:00:59.187518+0000","last_unstale":"2024-09-13T15:03:20.924191+0000","last_undegraded":"2024-09-13T15:03:20.924191+0000","last_fullsized":"2024-09-13T15:03:20.924191+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:14:59.602191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":358,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876729+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:03:03.876729+0000","last_peered":"2024-09-13T15:03:03.876729+0000","last_clean":"2024-09-13T15:03:03.876729+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:03:03.876729+0000","last_undegraded":"2024-09-13T15:03:03.876729+0000","last_fullsized":"2024-09-13T15:03:03.876729+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.920547+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:03:20.920547+0000","last_peered":"2024-09-13T15:03:20.920547+0000","last_clean":"2024-09-13T15:03:20.920547+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:03:20.920547+0000","last_undegraded":"2024-09-13T15:03:20.920547+0000","last_fullsized":"2024-09-13T15:03:20.920547+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878142+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T15:03:03.878142+0000","last_peered":"2024-09-13T15:03:03.878142+0000","last_clean":"2024-09-13T15:03:03.878142+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T15:03:03.878142+0000","last_undegraded":"2024-09-13T15:03:03.878142+0000","last_fullsized":"2024-09-13T15:03:03.878142+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":66,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877866+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:03:03.877866+0000","last_peered":"2024-09-13T15:03:03.877866+0000","last_clean":"2024-09-13T15:03:03.877866+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:03:03.877866+0000","last_undegraded":"2024-09-13T15:03:03.877866+0000","last_fullsized":"2024-09-13T15:03:03.877866+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":487,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.917264+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:03:20.917264+0000","last_peered":"2024-09-13T15:03:20.917264+0000","last_clean":"2024-09-13T15:03:20.917264+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:03:20.917264+0000","last_undegraded":"2024-09-13T15:03:20.917264+0000","last_fullsized":"2024-09-13T15:03:20.917264+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":542,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.917971+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:03:20.917971+0000","last_peered":"2024-09-13T15:03:20.917971+0000","last_clean":"2024-09-13T15:03:20.917971+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:03:20.917971+0000","last_undegraded":"2024-09-13T15:03:20.917971+0000","last_fullsized":"2024-09-13T15:03:20.917971+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.916433+0000","last_change":"2024-09-13T15:01:33.853899+0000","last_active":"2024-09-13T15:03:20.916433+0000","last_peered":"2024-09-13T15:03:20.916433+0000","last_clean":"2024-09-13T15:03:20.916433+0000","last_became_active":"2024-09-13T15:01:33.853515+0000","last_became_peered":"2024-09-13T15:01:33.853515+0000","last_unstale":"2024-09-13T15:03:20.916433+0000","last_undegraded":"2024-09-13T15:03:20.916433+0000","last_fullsized":"2024-09-13T15:03:20.916433+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:04.035329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":90,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.874534+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:03:03.874534+0000","last_peered":"2024-09-13T15:03:03.874534+0000","last_clean":"2024-09-13T15:03:03.874534+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:03:03.874534+0000","last_undegraded":"2024-09-13T15:03:03.874534+0000","last_fullsized":"2024-09-13T15:03:03.874534+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.925163+0000","last_change":"2024-09-13T15:00:58.194381+0000","last_active":"2024-09-13T15:03:20.925163+0000","last_peered":"2024-09-13T15:03:20.925163+0000","last_clean":"2024-09-13T15:03:20.925163+0000","last_became_active":"2024-09-13T15:00:58.194073+0000","last_became_peered":"2024-09-13T15:00:58.194073+0000","last_unstale":"2024-09-13T15:03:20.925163+0000","last_undegraded":"2024-09-13T15:03:20.925163+0000","last_fullsized":"2024-09-13T15:03:20.925163+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:54:00.089072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":566,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927609+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:03:20.927609+0000","last_peered":"2024-09-13T15:03:20.927609+0000","last_clean":"2024-09-13T15:03:20.927609+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:03:20.927609+0000","last_undegraded":"2024-09-13T15:03:20.927609+0000","last_fullsized":"2024-09-13T15:03:20.927609+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":345,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.876670+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T15:03:03.876670+0000","last_peered":"2024-09-13T15:03:03.876670+0000","last_clean":"2024-09-13T15:03:03.876670+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T15:03:03.876670+0000","last_undegraded":"2024-09-13T15:03:03.876670+0000","last_fullsized":"2024-09-13T15:03:03.876670+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":621,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.927762+0000","last_change":"2024-09-13T15:00:58.198846+0000","last_active":"2024-09-13T15:03:20.927762+0000","last_peered":"2024-09-13T15:03:20.927762+0000","last_clean":"2024-09-13T15:03:20.927762+0000","last_became_active":"2024-09-13T15:00:58.198544+0000","last_became_peered":"2024-09-13T15:00:58.198544+0000","last_unstale":"2024-09-13T15:03:20.927762+0000","last_undegraded":"2024-09-13T15:03:20.927762+0000","last_fullsized":"2024-09-13T15:03:20.927762+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:06.715499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":153,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.924364+0000","last_change":"2024-09-13T15:00:58.197368+0000","last_active":"2024-09-13T15:03:20.924364+0000","last_peered":"2024-09-13T15:03:20.924364+0000","last_clean":"2024-09-13T15:03:20.924364+0000","last_became_active":"2024-09-13T15:00:58.196459+0000","last_became_peered":"2024-09-13T15:00:58.196459+0000","last_unstale":"2024-09-13T15:03:20.924364+0000","last_undegraded":"2024-09-13T15:03:20.924364+0000","last_fullsized":"2024-09-13T15:03:20.924364+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:21:58.563761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":579,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.915746+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:03:20.915746+0000","last_peered":"2024-09-13T15:03:20.915746+0000","last_clean":"2024-09-13T15:03:20.915746+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:03:20.915746+0000","last_undegraded":"2024-09-13T15:03:20.915746+0000","last_fullsized":"2024-09-13T15:03:20.915746+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":580,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.931743+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:03:20.931743+0000","last_peered":"2024-09-13T15:03:20.931743+0000","last_clean":"2024-09-13T15:03:20.931743+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:03:20.931743+0000","last_undegraded":"2024-09-13T15:03:20.931743+0000","last_fullsized":"2024-09-13T15:03:20.931743+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878546+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T15:03:03.878546+0000","last_peered":"2024-09-13T15:03:03.878546+0000","last_clean":"2024-09-13T15:03:03.878546+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T15:03:03.878546+0000","last_undegraded":"2024-09-13T15:03:03.878546+0000","last_fullsized":"2024-09-13T15:03:03.878546+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:59.608331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":592,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.918543+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:03:20.918543+0000","last_peered":"2024-09-13T15:03:20.918543+0000","last_clean":"2024-09-13T15:03:20.918543+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:03:20.918543+0000","last_undegraded":"2024-09-13T15:03:20.918543+0000","last_fullsized":"2024-09-13T15:03:20.918543+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":65,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.925210+0000","last_change":"2024-09-13T15:00:59.191425+0000","last_active":"2024-09-13T15:03:20.925210+0000","last_peered":"2024-09-13T15:03:20.925210+0000","last_clean":"2024-09-13T15:03:20.925210+0000","last_became_active":"2024-09-13T15:00:59.189942+0000","last_became_peered":"2024-09-13T15:00:59.189942+0000","last_unstale":"2024-09-13T15:03:20.925210+0000","last_undegraded":"2024-09-13T15:03:20.925210+0000","last_fullsized":"2024-09-13T15:03:20.925210+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:45.702309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877938+0000","last_change":"2024-09-13T14:59:12.712394+0000","last_active":"2024-09-13T15:03:03.877938+0000","last_peered":"2024-09-13T15:03:03.877938+0000","last_clean":"2024-09-13T15:03:03.877938+0000","last_became_active":"2024-09-13T14:59:12.711381+0000","last_became_peered":"2024-09-13T14:59:12.711381+0000","last_unstale":"2024-09-13T15:03:03.877938+0000","last_undegraded":"2024-09-13T15:03:03.877938+0000","last_fullsized":"2024-09-13T15:03:03.877938+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:39:28.567717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":338,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877773+0000","last_change":"2024-09-13T15:01:33.852507+0000","last_active":"2024-09-13T15:03:03.877773+0000","last_peered":"2024-09-13T15:03:03.877773+0000","last_clean":"2024-09-13T15:03:03.877773+0000","last_became_active":"2024-09-13T15:01:33.852196+0000","last_became_peered":"2024-09-13T15:01:33.852196+0000","last_unstale":"2024-09-13T15:03:03.877773+0000","last_undegraded":"2024-09-13T15:03:03.877773+0000","last_fullsized":"2024-09-13T15:03:03.877773+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:28.217977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":67,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.878486+0000","last_change":"2024-09-13T15:00:59.184678+0000","last_active":"2024-09-13T15:03:03.878486+0000","last_peered":"2024-09-13T15:03:03.878486+0000","last_clean":"2024-09-13T15:03:03.878486+0000","last_became_active":"2024-09-13T15:00:59.184456+0000","last_became_peered":"2024-09-13T15:00:59.184456+0000","last_unstale":"2024-09-13T15:03:03.878486+0000","last_undegraded":"2024-09-13T15:03:03.878486+0000","last_fullsized":"2024-09-13T15:03:03.878486+0000","mapping_epoch":349,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:01.011514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":323,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.880137+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:03:03.880137+0000","last_peered":"2024-09-13T15:03:03.880137+0000","last_clean":"2024-09-13T15:03:03.880137+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:03:03.880137+0000","last_undegraded":"2024-09-13T15:03:03.880137+0000","last_fullsized":"2024-09-13T15:03:03.880137+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":623,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.928235+0000","last_change":"2024-09-13T15:00:58.197037+0000","last_active":"2024-09-13T15:03:20.928235+0000","last_peered":"2024-09-13T15:03:20.928235+0000","last_clean":"2024-09-13T15:03:20.928235+0000","last_became_active":"2024-09-13T15:00:58.196717+0000","last_became_peered":"2024-09-13T15:00:58.196717+0000","last_unstale":"2024-09-13T15:03:20.928235+0000","last_undegraded":"2024-09-13T15:03:20.928235+0000","last_fullsized":"2024-09-13T15:03:20.928235+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:16:43.954736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":589,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.932944+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:03:20.932944+0000","last_peered":"2024-09-13T15:03:20.932944+0000","last_clean":"2024-09-13T15:03:20.932944+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:03:20.932944+0000","last_undegraded":"2024-09-13T15:03:20.932944+0000","last_fullsized":"2024-09-13T15:03:20.932944+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":56,"reported_epoch":369,"state":"active+clean","last_fresh":"2024-09-13T15:03:03.877597+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T15:03:03.877597+0000","last_peered":"2024-09-13T15:03:03.877597+0000","last_clean":"2024-09-13T15:03:03.877597+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T15:03:03.877597+0000","last_undegraded":"2024-09-13T15:03:03.877597+0000","last_fullsized":"2024-09-13T15:03:03.877597+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":74,"reported_epoch":370,"state":"active+clean","last_fresh":"2024-09-13T15:03:20.925214+0000","last_change":"2024-09-13T15:00:58.194182+0000","last_active":"2024-09-13T15:03:20.925214+0000","last_peered":"2024-09-13T15:03:20.925214+0000","last_clean":"2024-09-13T15:03:20.925214+0000","last_became_active":"2024-09-13T15:00:58.193479+0000","last_became_peered":"2024-09-13T15:00:58.193479+0000","last_unstale":"2024-09-13T15:03:20.925214+0000","last_undegraded":"2024-09-13T15:03:20.925214+0000","last_fullsized":"2024-09-13T15:03:20.925214+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":349,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:43:34.966483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":13,"num_pg":16,"stat_sum":{"num_bytes":4948033560,"num_objects":75502,"num_object_clones":0,"num_object_copies":226506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":75502,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171496,"num_write_kb":7903873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14845030400,"data_stored":14845018184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125696,"ondisk_log_size":125696,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":5182,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234520,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2801400,"kb_used_data":1486576,"kb_used_omap":1,"kb_used_meta":1314814,"kb_avail":90943752,"statfs":{"total":95995035648,"available":93126402048,"internally_reserved":0,"allocated":1522253824,"data_stored":1520230618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1346369995},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496158,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2640816,"kb_used_data":1486632,"kb_used_omap":1,"kb_used_meta":1154174,"kb_avail":91104336,"statfs":{"total":95995035648,"available":93290840064,"internally_reserved":0,"allocated":1522311168,"data_stored":1520296093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1181874631},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757796,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3755112,"kb_used_data":2426080,"kb_used_omap":1,"kb_used_meta":1328958,"kb_avail":89990040,"statfs":{"total":95995035648,"available":92149800960,"internally_reserved":0,"allocated":2484305920,"data_stored":2482295076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1360853446},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019432,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3822328,"kb_used_data":2708272,"kb_used_omap":1,"kb_used_meta":1113982,"kb_avail":89922824,"statfs":{"total":95995035648,"available":92080971776,"internally_reserved":0,"allocated":2773270528,"data_stored":2771247342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1140718022},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650643,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171752,"kb_used_data":1283740,"kb_used_omap":4,"kb_used_meta":887995,"kb_avail":91573400,"statfs":{"total":95995035648,"available":93771161600,"internally_reserved":0,"allocated":1314549760,"data_stored":1312542774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":909307439},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879578,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3709680,"kb_used_data":2818084,"kb_used_omap":6,"kb_used_meta":891577,"kb_avail":90035472,"statfs":{"total":95995035648,"available":92196323328,"internally_reserved":0,"allocated":2885718016,"data_stored":2883703454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":912975417},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,3,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435595,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2601436,"kb_used_data":1504468,"kb_used_omap":5,"kb_used_meta":1096954,"kb_avail":91143716,"statfs":{"total":95995035648,"available":93331165184,"internally_reserved":0,"allocated":1540575232,"data_stored":1538575301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1123281249},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":348,"seq":1494648619039,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267236,"kb_used_data":1142812,"kb_used_omap":10,"kb_used_meta":124405,"kb_avail":92477916,"statfs":{"total":95995035648,"available":94697385984,"internally_reserved":0,"allocated":1170239488,"data_stored":1168232472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10675,"internal_metadata":127391309},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":921108480,"data_stored":921108480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1530662912,"data_stored":1530658840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2864447488,"data_stored":2864447488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1297612800,"data_stored":1297612800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2747338752,"data_stored":2747334680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2457403392,"data_stored":2457403392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1516699648,"data_stored":1516699648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":13,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1509756928,"data_stored":1509752856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:03:29.239 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:03:29.239 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:03:29.239 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 2, 3, 5, 6, 7, 0, 4] 2024-09-13T15:03:29.239 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd out 0 2024-09-13T15:03:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/247986670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:29.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:29.323 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:29.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:29.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.455 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:29.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:29.590 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:29.692 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:29.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:29.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:29.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:30.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:30.044 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:30.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:30.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:29 smithi042 ceph-mon[113926]: pgmap v1342: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2024-09-13T15:03:30.083 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:29 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:30.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:29 smithi042 ceph-mon[108570]: pgmap v1342: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2024-09-13T15:03:30.084 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:29 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:30.084 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:29] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-13T15:03:30.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:30.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:30.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:29 smithi159 ceph-mon[102519]: pgmap v1342: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 912 op/s 2024-09-13T15:03:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:29 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:03:30.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:30.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:30.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:30.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:30.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:30.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:30.470 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:30.519 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:30.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:30.643 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:30.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:30.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:30.804 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:30.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:30.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:30.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:30.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:30.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:31.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:31.096 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:03:31.096 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:03:31.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:31.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:31.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:31.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:31.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:31.371 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:31.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:31.511 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:31.613 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.644 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:31.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:31.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:31.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:31.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:31.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:31.917 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:31.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:32.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:31 smithi042 ceph-mon[113926]: pgmap v1343: 99 pgs: 99 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T15:03:32.017 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:31 smithi042 ceph-mon[108570]: pgmap v1343: 99 pgs: 99 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T15:03:32.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:03:32.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:32.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:32.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:32.197 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:32.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:31 smithi159 ceph-mon[102519]: pgmap v1343: 99 pgs: 99 active+clean; 4.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-13T15:03:32.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.344 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:32.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:32.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:03:32.445 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:32.482 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:32.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:32.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:32.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:32.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:32.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:32.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:32.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:32.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:32.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:32.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:32.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:33.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.121 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:33.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:33.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:33.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:33.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:33.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:33.495 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:33.532 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:33.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.534 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:33.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:33.694 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:33.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:33.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.831 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:33.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:33.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:33.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:33.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:34.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:34.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:34.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:33 smithi159 ceph-mon[102519]: pgmap v1344: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T15:03:34.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:33 smithi042 ceph-mon[113926]: pgmap v1344: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T15:03:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:33 smithi042 ceph-mon[108570]: pgmap v1344: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-13T15:03:34.363 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:34.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:34.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:34.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:34.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:34.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:34.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.693 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:34.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:34.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:34.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:34.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:34.954 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:34.970 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:03:34.985 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:34.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:34.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:35.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:35.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:35.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:35.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:35.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:35.346 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:35.346 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:03:35.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:35.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:35.522 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:35.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:35.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:35.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:35.673 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:35.679 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:35.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:35.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:35.861 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:35.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:35.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:35.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:36.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[108570]: pgmap v1345: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T15:03:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[108570]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T15:03:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[113926]: pgmap v1345: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T15:03:36.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:36.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:36.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[113926]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T15:03:36.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:36.144 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:36.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:35 smithi159 ceph-mon[102519]: pgmap v1345: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 708 op/s 2024-09-13T15:03:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:35 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:03:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2674401128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:03:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:35 smithi159 ceph-mon[102519]: osdmap e371: 8 total, 8 up, 8 in 2024-09-13T15:03:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.285 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:36.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:36.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:36.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:36.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:36.532 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:36.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:36.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:36.671 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:36.707 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:36.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:36.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:36.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:36.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:36.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:36.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:37.079 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:37.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:37.228 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:37.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:37.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:37.364 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.402 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:37.504 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:37.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:37.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:37.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:37.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:37.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:37.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:37.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:37.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:37.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:37.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:38.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:37 smithi159 ceph-mon[102519]: pgmap v1347: 99 pgs: 99 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-13T15:03:38.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:38.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:38.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:38.207 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:37 smithi042 ceph-mon[113926]: pgmap v1347: 99 pgs: 99 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-13T15:03:38.208 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:37 smithi042 ceph-mon[108570]: pgmap v1347: 99 pgs: 99 active+clean; 4.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-13T15:03:38.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:38.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:38.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:38.392 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:38.493 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:38.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.534 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:38.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:38.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:38.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:38.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:38.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:38.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:38.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:38.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:38.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:39.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.079 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:39.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:39.181 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:39.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:39.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:39 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:39 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:39.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:39.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:39.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:39 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:39.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:39.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:39.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:39.722 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:39.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:39.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:39.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:39.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:39.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:40.025 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:40.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:40.090 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:39] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-09-13T15:03:40.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:40.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:40.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.290 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:40.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:40.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:40 smithi042 ceph-mon[113926]: pgmap v1348: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:03:40.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:40 smithi042 ceph-mon[108570]: pgmap v1348: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:03:40.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:40.427 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:40.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:40 smithi159 ceph-mon[102519]: pgmap v1348: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-09-13T15:03:40.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:40.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:40.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:40.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:40.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:03:40.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:40.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:40.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:40.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:41.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.036 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:41.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:41.137 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:41.178 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:41.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:41.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.318 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:41.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:41.422 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:41.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:41.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:41.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:41.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:41.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:41.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:41.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:41.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:41.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:41.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:41.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:41.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:41.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:42.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.024 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:42.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:42.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.169 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:42.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:42.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:42.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:42.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:42 smithi042 ceph-mon[113926]: pgmap v1349: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T15:03:42.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:42 smithi042 ceph-mon[108570]: pgmap v1349: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T15:03:42.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:42.442 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:42.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:42 smithi159 ceph-mon[102519]: pgmap v1349: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-09-13T15:03:42.537 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:03:42.538 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:03:42.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:42.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:42.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:42.648 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:42 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:03:42.530+0000 7fcad2a32640 -1 mon.a@0(leader).osd e371 definitely_dead 0 2024-09-13T15:03:42.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:42.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:42.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:42.812 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:42.852 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:42.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:42.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:42.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:42.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:42.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:43.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:43.109 INFO:teuthology.orchestra.run.smithi042.stderr:marked out osd.0. 2024-09-13T15:03:43.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.116 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:43.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:43.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:43.239 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:43.340 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:43.355 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3773161437' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.356 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[113926]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T15:03:43.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3773161437' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.356 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:43 smithi042 ceph-mon[108570]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T15:03:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:43.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:43.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:43.486 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:43.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3773161437' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-09-13T15:03:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:43 smithi159 ceph-mon[102519]: Client client.admin marked osd.0 out, while it was still marked up 2024-09-13T15:03:43.511 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:03:43.511 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:03:43.511 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:03:43.512 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:43.513 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:43.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:43.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:43.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:43.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:43.733 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:43.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:43.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:43.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:43.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:43.981 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:43.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:44.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.104 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:44.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:44.227 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:44.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:44.350 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:44.452 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:44.476 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:44.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:44 smithi159 ceph-mon[102519]: pgmap v1350: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T15:03:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T15:03:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:44 smithi159 ceph-mon[102519]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:03:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/875054415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:44.569 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[113926]: pgmap v1350: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T15:03:44.569 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T15:03:44.569 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[113926]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:03:44.569 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/875054415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:44.569 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[108570]: pgmap v1350: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-13T15:03:44.570 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-09-13T15:03:44.570 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[108570]: osdmap e372: 8 total, 8 up, 7 in 2024-09-13T15:03:44.570 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/875054415' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:44.578 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:44.599 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:44.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:44.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:44.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:44.846 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:44.948 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:44.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:44.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:44.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:45.072 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:45.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:03:45.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:45.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:45.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:45.223 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:45.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:45.325 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:45 smithi042 ceph-mon[108570]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:03:45.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:45 smithi042 ceph-mon[108570]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:03:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:45 smithi042 ceph-mon[113926]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:03:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:45 smithi042 ceph-mon[113926]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:03:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:45.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.465 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:45 smithi159 ceph-mon[102519]: osdmap e373: 8 total, 8 up, 7 in 2024-09-13T15:03:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:45 smithi159 ceph-mon[102519]: osdmap e374: 8 total, 8 up, 7 in 2024-09-13T15:03:45.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:45.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:45.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.648 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:45.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:45.750 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:45.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:45.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:45.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:45.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:46.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:46.076 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:46.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:46.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:03:46.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:46.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[113926]: pgmap v1353: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T15:03:46.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[113926]: 13.9 starting backfill to osd.6 from (0'0,0'0] MAX to 371'11787 2024-09-13T15:03:46.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[113926]: 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'11651 2024-09-13T15:03:46.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[113926]: 13.5 starting backfill to osd.3 from (0'0,0'0] MAX to 371'11685 2024-09-13T15:03:46.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[113926]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:03:46.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[108570]: pgmap v1353: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T15:03:46.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[108570]: 13.9 starting backfill to osd.6 from (0'0,0'0] MAX to 371'11787 2024-09-13T15:03:46.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[108570]: 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'11651 2024-09-13T15:03:46.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[108570]: 13.5 starting backfill to osd.3 from (0'0,0'0] MAX to 371'11685 2024-09-13T15:03:46.232 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:46 smithi042 ceph-mon[108570]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:03:46.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:46.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:46.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:46.498 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:46 smithi159 ceph-mon[102519]: pgmap v1353: 99 pgs: 99 active+clean; 3.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 725 op/s 2024-09-13T15:03:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:46 smithi159 ceph-mon[102519]: 13.9 starting backfill to osd.6 from (0'0,0'0] MAX to 371'11787 2024-09-13T15:03:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:46 smithi159 ceph-mon[102519]: 13.3 starting backfill to osd.5 from (0'0,0'0] MAX to 371'11651 2024-09-13T15:03:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:46 smithi159 ceph-mon[102519]: 13.5 starting backfill to osd.3 from (0'0,0'0] MAX to 371'11685 2024-09-13T15:03:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:46 smithi159 ceph-mon[102519]: osdmap e375: 8 total, 8 up, 7 in 2024-09-13T15:03:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:46.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:46.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:46.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:46.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:46.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:46.877 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:46.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:46.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:46.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:47.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:47.050 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:47.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:47.177 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:47.242 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:47.264 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:47.278 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:47.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:47.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:47.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:47.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:47.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:47.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:47.720 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:47.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:47.748 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[113926]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:03:47.748 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[113926]: pgmap v1357: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 94 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 7267/178281 objects misplaced (4.076%); 1.9 MiB/s, 32 objects/s recovering 2024-09-13T15:03:47.748 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[113926]: 1.1 scrub starts 2024-09-13T15:03:47.748 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[113926]: 1.1 scrub ok 2024-09-13T15:03:47.749 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[108570]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:03:47.749 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[108570]: pgmap v1357: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 94 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 7267/178281 objects misplaced (4.076%); 1.9 MiB/s, 32 objects/s recovering 2024-09-13T15:03:47.749 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[108570]: 1.1 scrub starts 2024-09-13T15:03:47.749 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:47 smithi042 ceph-mon[108570]: 1.1 scrub ok 2024-09-13T15:03:47.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:47.858 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:47.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:47.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:47.960 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:47.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:47 smithi159 ceph-mon[102519]: osdmap e376: 8 total, 8 up, 7 in 2024-09-13T15:03:47.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:47 smithi159 ceph-mon[102519]: pgmap v1357: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 94 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 7267/178281 objects misplaced (4.076%); 1.9 MiB/s, 32 objects/s recovering 2024-09-13T15:03:47.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:47 smithi159 ceph-mon[102519]: 1.1 scrub starts 2024-09-13T15:03:47.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:47 smithi159 ceph-mon[102519]: 1.1 scrub ok 2024-09-13T15:03:48.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:48.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:48.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:48.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:48.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:48.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:48.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:48.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:48.529 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:48.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:48.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:48.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:48.701 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:48.803 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:48.837 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:48.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:48.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:48.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:49.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:49.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:49.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:49.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:49.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:49.394 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:49.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:49.536 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:49.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:49.590 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:03:49.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:03:49.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:49.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:49.747 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:03:49.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:49.808 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:49.909 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:49.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:49.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:49.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:50.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.083 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:50.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[113926]: pgmap v1358: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 10711/175395 objects misplaced (6.107%); 4.3 MiB/s, 71 objects/s recovering 2024-09-13T15:03:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3831319878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[108570]: pgmap v1358: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 10711/175395 objects misplaced (6.107%); 4.3 MiB/s, 71 objects/s recovering 2024-09-13T15:03:50.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3831319878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.100 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:49] "GET /metrics HTTP/1.1" 200 36531 "" "Prometheus/2.51.0" 2024-09-13T15:03:50.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:50.224 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:49 smithi159 ceph-mon[102519]: pgmap v1358: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.6 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 10711/175395 objects misplaced (6.107%); 4.3 MiB/s, 71 objects/s recovering 2024-09-13T15:03:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3831319878' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:49 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:03:50.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:50.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:50.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:50.499 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:50.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:50.601 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:50.637 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:50.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:50.739 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:50.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:50.881 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:50.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:50.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:50.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:51.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:51.044 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:03:51.044 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:03:51.045 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:03:51.045 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:03:51.045 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:03:51.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:51.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:03:51.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:51.045 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[113926]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:03:51.045 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:51.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:03:51.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:51.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:50 smithi042 ceph-mon[108570]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:03:51.070 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:51.172 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:51.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:51.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:50 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:03:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:50 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:03:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:50 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:03:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:50 smithi159 ceph-mon[102519]: osdmap e377: 8 total, 8 up, 7 in 2024-09-13T15:03:51.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:51.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:51.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:51.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:51.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:51.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:51.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:51.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:51.735 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:51.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:51.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.766 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:51.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:51.903 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:51.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:51.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:52.043 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:52.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[113926]: pgmap v1360: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 10711/173790 objects misplaced (6.163%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T15:03:52.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[108570]: pgmap v1360: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 10711/173790 objects misplaced (6.163%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T15:03:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:52.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:52.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:51 smithi159 ceph-mon[102519]: pgmap v1360: 99 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 3 peering, 93 active+clean; 3.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 10711/173790 objects misplaced (6.163%); 3.6 MiB/s, 58 objects/s recovering 2024-09-13T15:03:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:03:52.281 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:52.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:52.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:52.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:52.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:52.555 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:52.596 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:52.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:52.735 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:52.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:52.873 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:52.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:52.974 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:03:53.111 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:53.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:53.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:53.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:53.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:53.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:53.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:53.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:53.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:53.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:53.814 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.854 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:53.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:53.965 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:53 smithi042 ceph-mon[108570]: pgmap v1361: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 9647/164319 objects misplaced (5.871%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:03:53.965 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:53 smithi042 ceph-mon[113926]: pgmap v1361: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 9647/164319 objects misplaced (5.871%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:03:53.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:53.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:54.091 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:54.127 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:54.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:53 smithi159 ceph-mon[102519]: pgmap v1361: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 861 op/s; 9647/164319 objects misplaced (5.871%); 7.4 MiB/s, 118 objects/s recovering 2024-09-13T15:03:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.272 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:54.373 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:54.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.438 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:54.504 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:03:54.504 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:03:54.538 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:54.544 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:03:54.544 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:03:54.577 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:54.584 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:03:54.585 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:03:54.620 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:03:54.620 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:03:54.653 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:03:54.653 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:03:54.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:54.684 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:54.689 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:03:54.690 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:03:54.725 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:03:54.725 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:03:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:54.760 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:03:54.760 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:03:54.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:54.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:54.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:54.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:54.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:55.014 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4000669712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4000669712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:55.116 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.155 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4000669712' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:03:55.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:55.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:55.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:55.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:55.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:55.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:55.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:03:55.715 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:55.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:55.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:55.956 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:55.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:55.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:55.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:56.070 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[113926]: pgmap v1362: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 8955/161562 objects misplaced (5.543%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T15:03:56.070 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.070 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.071 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[108570]: pgmap v1362: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 8955/161562 objects misplaced (5.543%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T15:03:56.071 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.071 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:56.141 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:56.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:56.243 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:55 smithi159 ceph-mon[102519]: pgmap v1362: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 685 op/s; 8955/161562 objects misplaced (5.543%); 8.8 MiB/s, 141 objects/s recovering 2024-09-13T15:03:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:56.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:56.374 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:56.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:56.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:56.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:56.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:03:56.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:56.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:03:56.770 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:56.788 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:56.818 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:56.849 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:03:56.920 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:56.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:56.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:56.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:57.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:57.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:57.197 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:57.230 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:03:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:03:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:03:57.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:57.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:57.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:57.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:57.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:57.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:57.775 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:57.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:57.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:57.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:03:57.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:57.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:57.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:03:57.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:03:58.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:58.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:58.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:57 smithi042 ceph-mon[113926]: pgmap v1363: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 7928/151875 objects misplaced (5.220%); 10 MiB/s, 164 objects/s recovering 2024-09-13T15:03:58.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:57 smithi042 ceph-mon[108570]: pgmap v1363: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 7928/151875 objects misplaced (5.220%); 10 MiB/s, 164 objects/s recovering 2024-09-13T15:03:58.189 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:58.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.244 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:57 smithi159 ceph-mon[102519]: pgmap v1363: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.1 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 879 op/s; 7928/151875 objects misplaced (5.220%); 10 MiB/s, 164 objects/s recovering 2024-09-13T15:03:58.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:58.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.525 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:58.627 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:03:58.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:58.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:03:58.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:58.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:58.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:58.955 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:58.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:59.057 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.104 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:59.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:59.348 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:03:59.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.379 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:03:59.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:03:59.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:03:59.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:03:59.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:03:59.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:03:59.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:03:59.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:03:59.842 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:03:59] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.51.0" 2024-09-13T15:03:59.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:03:59.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:03:59.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:03:59.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:03:59.954 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:04:00.032 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:00.062 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:00.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[108570]: pgmap v1364: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 7270/147366 objects misplaced (4.933%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3454603902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[113926]: pgmap v1364: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 7270/147366 objects misplaced (4.933%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3454603902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:03:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.164 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:00.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:59 smithi159 ceph-mon[102519]: pgmap v1364: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 933 op/s; 7270/147366 objects misplaced (4.933%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3454603902' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:03:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:04:00.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:00.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:00.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:00.436 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:00.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:00.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:00.528 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:00.528 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:00.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:00.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.617 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:00.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:00.718 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:00.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:00.859 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:00.897 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:00.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:00.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:00.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:01.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.033 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:01.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:01.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:00 smithi042 ceph-mon[108570]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:04:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:00 smithi042 ceph-mon[113926]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:04:01.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:01.162 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:01.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:04:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:00 smithi159 ceph-mon[102519]: osdmap e378: 8 total, 8 up, 7 in 2024-09-13T15:04:01.264 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:01.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:04:01.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:01.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:01.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:01.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:01.582 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:01.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:01.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:01.721 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:01.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:01.823 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:01.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:01.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:01.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:01.964 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:02.001 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:02.104 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:02.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:02.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:01 smithi159 ceph-mon[102519]: pgmap v1366: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 7270/145779 objects misplaced (4.987%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:01 smithi042 ceph-mon[108570]: pgmap v1366: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 7270/145779 objects misplaced (4.987%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:02.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:01 smithi042 ceph-mon[113926]: pgmap v1366: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 3.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 7270/145779 objects misplaced (4.987%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:04:02.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:02.419 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:02.521 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:02.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:02.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:02.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:02.945 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:02.976 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:02.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:02.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:03.078 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:03.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.116 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:03.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:03.217 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:03.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:03.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:03.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:03.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:03.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:03.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:03.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:03.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:03.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:03.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:03.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:03.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:03.905 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:03.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:03.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:04.048 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:04.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:04.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:04.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:04.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:03 smithi159 ceph-mon[102519]: pgmap v1367: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6284/136149 objects misplaced (4.616%); 12 MiB/s, 185 objects/s recovering 2024-09-13T15:04:04.319 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:04.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:03 smithi042 ceph-mon[113926]: pgmap v1367: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6284/136149 objects misplaced (4.616%); 12 MiB/s, 185 objects/s recovering 2024-09-13T15:04:04.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:03 smithi042 ceph-mon[108570]: pgmap v1367: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 938 op/s; 6284/136149 objects misplaced (4.616%); 12 MiB/s, 185 objects/s recovering 2024-09-13T15:04:04.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:04.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:04.465 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:04.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.508 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:04.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:04.609 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:04.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:04.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:04.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:04.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:04.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:04.926 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:04.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:04.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:05.028 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:05.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:05.168 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:05.193 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:05.295 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:05.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:05.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:05.469 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:05.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:05.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:05.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:05.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:05.733 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:05.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:05.835 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:05.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:05.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:05.984 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:06.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:06.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:06.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:06.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:05 smithi159 ceph-mon[102519]: pgmap v1368: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 5680/133452 objects misplaced (4.256%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:04:06.342 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:05 smithi042 ceph-mon[113926]: pgmap v1368: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 5680/133452 objects misplaced (4.256%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:04:06.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:05 smithi042 ceph-mon[108570]: pgmap v1368: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 936 op/s; 5680/133452 objects misplaced (4.256%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:04:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:06.373 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:06.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:06.475 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:06.513 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:06.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:06.617 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:06.659 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:06.668 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:06.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:04:06.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:06.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:06.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:06.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:06.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:06.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:07.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:07.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:07.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:07.219 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:07.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:07.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:07.457 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.489 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:07.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:07.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:07.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:07.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.767 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:07.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:07.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:07.906 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:07.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:07.976 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:08.009 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:08.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.050 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:08.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:08.151 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.183 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:07 smithi042 ceph-mon[108570]: pgmap v1369: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 4705/124116 objects misplaced (3.791%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:04:08.184 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:07 smithi042 ceph-mon[113926]: pgmap v1369: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 4705/124116 objects misplaced (3.791%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:04:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.191 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:08.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:07 smithi159 ceph-mon[102519]: pgmap v1369: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 4705/124116 objects misplaced (3.791%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:04:08.292 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:08.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:08.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:08.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:08.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:08.486 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:08.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:08.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:08.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:08.706 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:08.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:08.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:08.846 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:08.877 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:08.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:08.978 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:08.995 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:04:09.013 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:09.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:09.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:09.115 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:09.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:09.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:09.291 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:09.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.431 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:04:09.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:09.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:09.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:09.708 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:09.810 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:09.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:09] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T15:04:09.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:09.958 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:09.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:09.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:09.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:10.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:10.134 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:10.236 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:09 smithi159 ceph-mon[102519]: pgmap v1370: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 4123/119526 objects misplaced (3.449%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:09 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:09 smithi159 ceph-mon[102519]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:04:10.271 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:10.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:10.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[108570]: pgmap v1370: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 4123/119526 objects misplaced (3.449%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[108570]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[113926]: pgmap v1370: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 927 op/s; 4123/119526 objects misplaced (3.449%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1133817367' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:04:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:09 smithi042 ceph-mon[113926]: osdmap e379: 8 total, 8 up, 7 in 2024-09-13T15:04:10.373 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:10.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:10.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:10.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:10.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:10.700 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:10.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:10.802 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:10.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:10.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:10.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:10.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:10.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:10.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:11.073 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:11.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:11.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:11.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:11.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:11.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:11.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:11.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:11.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:11.500 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:11.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:11.636 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:11.676 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:11.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:11.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:11.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:11.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:11.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:11.956 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:11.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:12.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:12.095 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:12.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:12.179 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:12 smithi042 ceph-mon[113926]: pgmap v1372: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 4123/117720 objects misplaced (3.502%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:12.179 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:12 smithi042 ceph-mon[108570]: pgmap v1372: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 4123/117720 objects misplaced (3.502%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:12.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:12.226 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:12.328 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:12.365 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:12.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:12.467 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:12 smithi159 ceph-mon[102519]: pgmap v1372: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 4123/117720 objects misplaced (3.502%); 12 MiB/s, 196 objects/s recovering 2024-09-13T15:04:12.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:12.611 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:12.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:04:12.715 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:12.716 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:12.754 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:12.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:12.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:12.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:12.919 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:12.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:13.011 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:04:13.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.055 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:13.157 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:13.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:13.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:13.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/993833343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2228444599' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:13.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/993833343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:13 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2228444599' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:13.428 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.452 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:13.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/993833343' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:13 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:04:13.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:13 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2228444599' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:13.554 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:13.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:13.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:13.705 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:13.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:13.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:13.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:13.954 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:13.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:13.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:14.056 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:14.082 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:14.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:14.184 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:14.212 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:14.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:14.338 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:14.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[113926]: pgmap v1373: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 3247/107802 objects misplaced (3.012%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:04:14.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:14.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[113926]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:04:14.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[108570]: pgmap v1373: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 3247/107802 objects misplaced (3.012%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:04:14.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:14.350 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:14 smithi042 ceph-mon[108570]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:04:14.440 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:14 smithi159 ceph-mon[102519]: pgmap v1373: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 944 op/s; 3247/107802 objects misplaced (3.012%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:04:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:04:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:14 smithi159 ceph-mon[102519]: osdmap e380: 8 total, 8 up, 7 in 2024-09-13T15:04:14.564 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.586 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:14.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:14.812 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:14.835 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:14.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:14.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:14.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:15.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:15.087 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:15.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:15.189 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:15.230 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:15.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:15 smithi042 ceph-mon[113926]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:04:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:15 smithi042 ceph-mon[108570]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:04:15.371 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:15.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:15.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:15 smithi159 ceph-mon[102519]: osdmap e381: 8 total, 8 up, 7 in 2024-09-13T15:04:15.511 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:15.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:15.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:15.647 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:15.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:15.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:15.784 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:214620 13.b 13:d67d57df:::benchmark_data_smithi042_286876_object91119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:04:15.865734+0000", 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.014508263, 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014536719, 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:15.883 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214620 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865734+0000", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:15.884 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865734+0000", 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865735+0000", 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2729999999999999e-06 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865739+0000", 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0169999999999998e-06 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.885 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865752+0000", 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3158e-05 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865761+0000", 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.619e-06 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:15.886 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865807+0000", 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5775999999999997e-05 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.865935+0000", 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000127837 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.866559+0000", 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00062465600000000002 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.887 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.872660+0000", 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061004989999999997 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874200+0000", 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001540154 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.888 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:214622 13.5 e382/380 13:a7a6f23c:::benchmark_data_smithi042_286876_object91121:head v 382'13365, mlcod=382'13359)", 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:04:15.873596+0000", 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0066463690000000001, 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0067520949999999996, 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.889 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873596+0000", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873596+0000", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873596+0000", 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0500000000000003e-07 2024-09-13T15:04:15.890 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873602+0000", 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2310000000000004e-06 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873611+0000", 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9420000000000001e-06 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:15.891 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873617+0000", 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2020000000000001e-06 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873676+0000", 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8776000000000001e-05 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.873694+0000", 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8091999999999999e-05 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.892 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880146+0000", 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064520070000000001 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:214626 13.2 13:47a6924e:::benchmark_data_smithi042_286876_object91125:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:04:15.874350+0000", 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0058920650000000001, 2024-09-13T15:04:15.893 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060617900000000001, 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 214626 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874350+0000", 2024-09-13T15:04:15.894 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874350+0000", 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874352+0000", 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7150000000000001e-06 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:15.895 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874356+0000", 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5820000000000003e-06 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874368+0000", 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1491999999999999e-05 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874376+0000", 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0930000000000003e-06 2024-09-13T15:04:15.896 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874438+0000", 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2169999999999996e-05 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874565+0000", 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012724500000000001 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:04:15.897 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.874688+0000", 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012253599999999999 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45788.0:214620 13.b e382/354 ondisk, result = 0)", 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:04:15.880228+0000", 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.4163e-05, 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000237316, 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:15.898 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880228+0000", 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880228+0000", 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.899 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880229+0000", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9199999999999999e-07 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880235+0000", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6350000000000003e-06 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880247+0000", 2024-09-13T15:04:15.900 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2774e-05 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880255+0000", 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8169999999999999e-06 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.901 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.902 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:15.902 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880342+0000", 2024-09-13T15:04:15.902 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7075000000000002e-05 2024-09-13T15:04:15.902 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.902 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880351+0000", 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2799999999999992e-06 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880465+0000", 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113843 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:15.903 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45788.0:214626 13.2 e382/354)", 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:04:15.880352+0000", 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 4294967295.9998903, 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017664199999999999, 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "queued for pg", 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880352+0000", 2024-09-13T15:04:15.904 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880352+0000", 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880352+0000", 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5400000000000001e-07 2024-09-13T15:04:15.905 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880357+0000", 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1089999999999997e-06 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880368+0000", 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0757000000000001e-05 2024-09-13T15:04:15.906 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:04:15.880390+0000", 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2149000000000001e-05 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 5 2024-09-13T15:04:15.907 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:04:15.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:15.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:15.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:15.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:16.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.077 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:16.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:16.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:16.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:16.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:16.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:16 smithi042 ceph-mon[113926]: pgmap v1376: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 2738/105072 objects misplaced (2.606%); 9.8 MiB/s, 157 objects/s recovering 2024-09-13T15:04:16.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:16 smithi042 ceph-mon[113926]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:04:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:16 smithi042 ceph-mon[108570]: pgmap v1376: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 2738/105072 objects misplaced (2.606%); 9.8 MiB/s, 157 objects/s recovering 2024-09-13T15:04:16.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:16 smithi042 ceph-mon[108570]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:04:16.447 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:16.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:16 smithi159 ceph-mon[102519]: pgmap v1376: 99 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 96 active+clean; 2.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 802 op/s; 2738/105072 objects misplaced (2.606%); 9.8 MiB/s, 157 objects/s recovering 2024-09-13T15:04:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:16 smithi159 ceph-mon[102519]: osdmap e382: 8 total, 8 up, 7 in 2024-09-13T15:04:16.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:16.617 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:16.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:16.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:16.749 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:16.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:16.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:16.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:16.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:16.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:16.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:17.036 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:17.055 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:17.138 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:17.170 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:17.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.415 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:17.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:17.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.732 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:17.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:17.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:17.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:17.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:18.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:18.108 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:18.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:04:18.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:18.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:18.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:18 smithi042 ceph-mon[108570]: pgmap v1378: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 1234/98286 objects misplaced (1.256%); 24 MiB/s, 379 objects/s recovering 2024-09-13T15:04:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:18 smithi042 ceph-mon[113926]: pgmap v1378: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 1234/98286 objects misplaced (1.256%); 24 MiB/s, 379 objects/s recovering 2024-09-13T15:04:18.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:18.424 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:18 smithi159 ceph-mon[102519]: pgmap v1378: 99 pgs: 2 active+remapped+backfilling, 97 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.08k op/s; 1234/98286 objects misplaced (1.256%); 24 MiB/s, 379 objects/s recovering 2024-09-13T15:04:18.526 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:18.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:18.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:18.706 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.707 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:18.807 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:18.840 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:18.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:18.942 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:18.976 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:18.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:19.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:19.133 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T15:04:19.141 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:19.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:19.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.261 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:19.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:19.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:19.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:19.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:19.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:19.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:19.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:19.688 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:19.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:19.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.825 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:19.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:19.926 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:19.967 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:19.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:19.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:20.027 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:19] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T15:04:20.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:20.215 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.244 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:20.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:20.301 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:20 smithi159 ceph-mon[102519]: pgmap v1379: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 835/95208 objects misplaced (0.877%); 22 MiB/s, 359 objects/s recovering 2024-09-13T15:04:20.345 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:20.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:20 smithi042 ceph-mon[113926]: pgmap v1379: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 835/95208 objects misplaced (0.877%); 22 MiB/s, 359 objects/s recovering 2024-09-13T15:04:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:20 smithi042 ceph-mon[108570]: pgmap v1379: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 699 op/s; 835/95208 objects misplaced (0.877%); 22 MiB/s, 359 objects/s recovering 2024-09-13T15:04:20.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:20.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:20.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:20.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:20.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:04:20.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:20.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:20.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:20.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:20.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:20.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:20.849 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:20.862 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:20.934 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:20.963 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:20.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:20.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:21.065 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:21.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.243 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:21.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:21.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:21.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:21.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:21.476 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:21.520 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:21.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:21.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:21.661 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:21.720 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:21.762 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:04:21.762 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:21.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:21.789 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:04:21.789 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:21.798 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:21.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:04:21.816 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:21.842 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:04:21.842 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:21.879 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:04:21.879 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:21.899 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:21.912 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:04:21.912 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:21.938 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:21.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:21.946 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:04:21.947 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:21.971 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:04:21.971 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:22.040 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:22.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[113926]: pgmap v1380: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 835/93357 objects misplaced (0.894%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3365842102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[108570]: pgmap v1380: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 835/93357 objects misplaced (0.894%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3365842102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:22.179 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:22.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:21 smithi159 ceph-mon[102519]: pgmap v1380: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 507 op/s; 835/93357 objects misplaced (0.894%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:04:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3365842102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:22.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.351 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:22.452 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:22.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:22.604 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:22.644 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:22.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:22.784 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:22.886 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:22.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:22.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:23.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:23.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:23.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:23.180 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:23.283 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:23.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:23.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:23.453 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:23.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:23.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.592 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:23.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:23.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:23.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.865 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:23.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:23.967 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:23.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:24.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:23 smithi042 ceph-mon[113926]: pgmap v1381: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 297/83616 objects misplaced (0.355%); 16 MiB/s, 250 objects/s recovering 2024-09-13T15:04:24.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:23 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]} : dispatch 2024-09-13T15:04:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:23 smithi042 ceph-mon[108570]: pgmap v1381: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 297/83616 objects misplaced (0.355%); 16 MiB/s, 250 objects/s recovering 2024-09-13T15:04:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:23 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]} : dispatch 2024-09-13T15:04:24.101 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:24.136 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:24.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:24.238 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:23 smithi159 ceph-mon[102519]: pgmap v1381: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 824 op/s; 297/83616 objects misplaced (0.355%); 16 MiB/s, 250 objects/s recovering 2024-09-13T15:04:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:23 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]} : dispatch 2024-09-13T15:04:24.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:24.376 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:24.417 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:24.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:24.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:24.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:24.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:24.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:24.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:24.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:24.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:24.938 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:24.969 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:24.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:24.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:25.064 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T15:04:25.065 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:24 smithi042 ceph-mon[113926]: osdmap e383: 8 total, 8 up, 7 in 2024-09-13T15:04:25.065 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T15:04:25.065 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:24 smithi042 ceph-mon[108570]: osdmap e383: 8 total, 8 up, 7 in 2024-09-13T15:04:25.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:25.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:25.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:25.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:25.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:25.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-09-13T15:04:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:24 smithi159 ceph-mon[102519]: osdmap e383: 8 total, 8 up, 7 in 2024-09-13T15:04:25.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:25.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:25.445 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:25.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:25.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:25.568 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:25.590 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:25.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:25.692 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:25.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:25.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:25.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:25.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:25.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:25.943 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:25.975 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:25.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:25.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:25 smithi042 ceph-mon[108570]: pgmap v1383: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 297/80958 objects misplaced (0.367%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:26.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:25 smithi042 ceph-mon[108570]: osdmap e384: 8 total, 8 up, 7 in 2024-09-13T15:04:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:25 smithi042 ceph-mon[113926]: pgmap v1383: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 297/80958 objects misplaced (0.367%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:25 smithi042 ceph-mon[113926]: osdmap e384: 8 total, 8 up, 7 in 2024-09-13T15:04:26.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:26.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:26.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:25 smithi159 ceph-mon[102519]: pgmap v1383: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 787 op/s; 297/80958 objects misplaced (0.367%); 14 MiB/s, 224 objects/s recovering 2024-09-13T15:04:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:25 smithi159 ceph-mon[102519]: osdmap e384: 8 total, 8 up, 7 in 2024-09-13T15:04:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.253 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:26.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:26.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.396 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:26.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:26.492 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:26.492 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:26.497 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:26.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:26.639 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:26.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:26.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:26.918 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:26.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:26.953 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:26.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:27.009 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:26 smithi042 ceph-mon[113926]: osdmap e385: 8 total, 8 up, 7 in 2024-09-13T15:04:27.010 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:26 smithi042 ceph-mon[108570]: osdmap e385: 8 total, 8 up, 7 in 2024-09-13T15:04:27.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:27.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:27.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:27.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:27.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:26 smithi159 ceph-mon[102519]: osdmap e385: 8 total, 8 up, 7 in 2024-09-13T15:04:27.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:27.365 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:27.467 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:27.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:27.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:27.884 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:27.890 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:27 smithi042 ceph-mon[108570]: pgmap v1386: 99 pgs: 99 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 7.5 MiB/s, 120 objects/s recovering 2024-09-13T15:04:27.890 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:27 smithi042 ceph-mon[113926]: pgmap v1386: 99 pgs: 99 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 7.5 MiB/s, 120 objects/s recovering 2024-09-13T15:04:27.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:27.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:28.027 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:28.063 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:28.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:28.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:27 smithi159 ceph-mon[102519]: pgmap v1386: 99 pgs: 99 active+clean; 1.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 7.5 MiB/s, 120 objects/s recovering 2024-09-13T15:04:28.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:28.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:28.460 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:28.490 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:28.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:28.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:28.622 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:28.724 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:28.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:28.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:28.860 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:04:28.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:28.904 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:28.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.113 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3881524493' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.113 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.113 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3881524493' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.113 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:04:29.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.187 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3881524493' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:29.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:29.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:29.567 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:29.605 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:29.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:29.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:29.848 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:29.869 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T15:04:29.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:29.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:29.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:29.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:30.024 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:30.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: pgmap v1387: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 2.9 MiB/s, 46 objects/s recovering 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: osdmap e386: 8 total, 8 up, 7 in 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2748040731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:29] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.51.0" 2024-09-13T15:04:30.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: pgmap v1387: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 2.9 MiB/s, 46 objects/s recovering 2024-09-13T15:04:30.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:30.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:30.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: osdmap e386: 8 total, 8 up, 7 in 2024-09-13T15:04:30.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2748040731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.126 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.155 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:30.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: pgmap v1387: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 817 op/s; 2.9 MiB/s, 46 objects/s recovering 2024-09-13T15:04:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: osdmap e386: 8 total, 8 up, 7 in 2024-09-13T15:04:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2748040731' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:04:30.256 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:30.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:30.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:30.332 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:30.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:30.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:30.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:30.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:30.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:30.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.710 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:30.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:30.782 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:04:30.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:30.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:30.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:30.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:30.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:30.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:31.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:30 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:04:31.041 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:30 smithi042 ceph-mon[108570]: osdmap e387: 8 total, 8 up, 7 in 2024-09-13T15:04:31.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:30 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:04:31.041 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:30 smithi042 ceph-mon[113926]: osdmap e387: 8 total, 8 up, 7 in 2024-09-13T15:04:31.094 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:31.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:31.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:30 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:04:31.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:30 smithi159 ceph-mon[102519]: osdmap e387: 8 total, 8 up, 7 in 2024-09-13T15:04:31.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:31.366 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:31.392 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:31.494 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:31.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:31.603 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:31.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:31.671 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:31.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:31.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:31.807 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:31.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:31.943 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:31.975 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:31.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:32.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:31 smithi042 ceph-mon[108570]: pgmap v1390: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:04:32.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:31 smithi042 ceph-mon[113926]: pgmap v1390: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:04:32.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:32.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.124 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:04:32.198 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:32.214 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:32.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:31 smithi159 ceph-mon[102519]: pgmap v1390: 99 pgs: 99 active+clean; 1.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 2.9 MiB/s, 47 objects/s recovering 2024-09-13T15:04:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.251 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:32.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:32.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:32.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:32.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:32.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:32.521 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:32.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:32.622 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:32.654 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:32.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:32.789 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:32.891 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:32.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:32.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:33.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:33.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:33.169 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:33.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:33.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:33.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:33.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:33.395 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:33.396 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:33.397 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:33.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:33.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:33.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:33.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:33.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:33.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:33.753 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:33.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:33.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:33.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:33.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:33.992 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:34.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:34.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:33 smithi159 ceph-mon[102519]: pgmap v1391: 99 pgs: 99 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:04:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1510487589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:34.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:34.298 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:33 smithi042 ceph-mon[113926]: pgmap v1391: 99 pgs: 99 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:04:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1510487589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:33 smithi042 ceph-mon[108570]: pgmap v1391: 99 pgs: 99 active+clean; 1.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:04:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1510487589' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:34.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:34.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:34.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:34.575 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:34.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:34.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:34.826 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:34.868 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:34.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:34.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:34.969 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:35.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:35.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:35.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:35.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:35.421 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:35.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:35.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:35.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:35.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:35.713 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:35.815 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:35.847 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:35.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:35.993 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:35.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.068 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:35 smithi042 ceph-mon[113926]: pgmap v1392: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T15:04:36.068 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.068 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:35 smithi042 ceph-mon[108570]: pgmap v1392: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T15:04:36.069 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.094 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.127 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:36.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:36.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:35 smithi159 ceph-mon[102519]: pgmap v1392: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 750 op/s 2024-09-13T15:04:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:36.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:36.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:36.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:36.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:36.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:36.502 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:36.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:36.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:36.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:36.688 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:36.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:36.790 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:36.820 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:36.922 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:36.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:36.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:36.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86125 12.11 12:8a0d8b36:::benchmark_data_smithi042_260592_object86124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 404.77361750799997, 2024-09-13T15:04:37.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3037125899999999, 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86125 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:04:37.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151484+0000", 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3399999999999999e-07 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151505+0000", 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0979e-05 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151513+0000", 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3180000000000001e-06 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151518+0000", 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4040000000000003e-06 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152398+0000", 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00087995399999999996 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152404+0000", 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1680000000000001e-06 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416477+0000", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2640725930000001 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416584+0000", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010674 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.416736+0000", 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015215300000000001 2024-09-13T15:04:37.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422773+0000", 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0060373429999999997 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445727+0000", 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022953881999999998 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.042 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453059+0000", 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0073320290000000003 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.453124+0000", 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5162999999999999e-05 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.455196+0000", 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020723299999999998 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.043 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86135 12.11 12:8c7021b7:::benchmark_data_smithi042_260592_object86134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 404.77357649999999, 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3064091740000001, 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86135 2024-09-13T15:04:37.044 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151525+0000", 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4000000000000002e-07 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151544+0000", 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8833000000000001e-05 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151551+0000", 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7279999999999998e-06 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151556+0000", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3129999999999998e-06 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152425+0000", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00086871300000000002 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152431+0000", 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.144e-06 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.417930+0000", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2654990859999999 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418028+0000", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7684999999999997e-05 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.418152+0000", 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012413100000000001 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422904+0000", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047515930000000001 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445774+0000", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.022870715 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456350+0000", 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010575721 2024-09-13T15:04:37.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.456409+0000", 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9438000000000002e-05 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.457934+0000", 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001524634 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86235 12.11 12:8a3e8959:::benchmark_data_smithi042_260592_object86234:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 404.77353899899998, 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3274169330000001, 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86235 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151562+0000", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151563+0000", 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2300000000000002e-07 2024-09-13T15:04:37.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151587+0000", 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3813000000000001e-05 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151593+0000", 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.844e-06 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151599+0000", 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2009999999999998e-06 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152452+0000", 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085325899999999998 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152458+0000", 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.049e-06 2024-09-13T15:04:37.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.422991+0000", 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.270532545 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423159+0000", 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016851999999999999 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.423348+0000", 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018931899999999999 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.428067+0000", 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047184879999999998 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445827+0000", 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.017759664000000001 2024-09-13T15:04:37.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474222+0000", 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.028395482999999999 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.474279+0000", 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7204999999999999e-05 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.478979+0000", 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047001200000000003 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:86275 12.11 12:8d556199:::benchmark_data_smithi042_260592_object86274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 404.77349642799999, 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3296465079999999, 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.058 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 86275 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151605+0000", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4999999999999998e-07 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151627+0000", 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1914999999999999e-05 2024-09-13T15:04:37.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151634+0000", 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6479999999999997e-06 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.151639+0000", 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.578e-06 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152479+0000", 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00084047799999999997 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:52.152485+0000", 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.207e-06 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424679+0000", 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2721941349999999 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.424837+0000", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015725900000000001 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.425047+0000", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000210137 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.429698+0000", 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0046507639999999999 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.445901+0000", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016203077999999999 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479037+0000", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.033136456000000002 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.479191+0000", 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015432 2024-09-13T15:04:37.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T14:57:53.481251+0000", 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0020600829999999999 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45599.0:217201 12.2 12:4755d4bb:::benchmark_data_smithi042_260592_object100310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e335)", 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 273.94049362200002, 2024-09-13T15:04:37.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.144661345, 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45599", 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 217201 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984608+0000", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984610+0000", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1210000000000001e-06 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984616+0000", 2024-09-13T15:04:37.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0839999999999997e-06 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984628+0000", 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2517000000000001e-05 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.984636+0000", 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6699999999999994e-06 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985073+0000", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00043719400000000002 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985220+0000", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014707300000000001 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,7", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:02.985790+0000", 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00056957299999999998 2024-09-13T15:04:37.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.123004+0000", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.137214313 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126034+0000", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030292850000000001 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.126049+0000", 2024-09-13T15:04:37.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5109999999999999e-05 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.127662+0000", 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001613225 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:00:03.129269+0000", 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0016071799999999999 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:55028 13.b e353/344 13:da56cd9d:::benchmark_data_smithi042_286876_object55027:head v 353'3406, mlcod=353'3404)", 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.49152007199999, 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25938409099999998, 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:04:37.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433581+0000", 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433582+0000", 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6000000000000002e-07 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433693+0000", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011040300000000001 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.433704+0000", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1885999999999999e-05 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687498+0000", 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25379304400000002 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687603+0000", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105482 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687629+0000", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6492e-05 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692925+0000", 2024-09-13T15:04:37.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052956619999999996 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692965+0000", 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0262000000000002e-05 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:55062 13.6 e353/344 13:629fd30b:::benchmark_data_smithi042_286876_object55061:head v 353'3438, mlcod=353'3436)", 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:04:37.076 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 187.47058010399999, 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23819004399999999, 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454521+0000", 2024-09-13T15:04:37.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454531+0000", 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0142999999999999e-05 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.454596+0000", 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4306000000000004e-05 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687499+0000", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.232902998 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687516+0000", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6940999999999999e-05 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687709+0000", 2024-09-13T15:04:37.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019348499999999999 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.687727+0000", 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7561e-05 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692674+0000", 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0049475880000000002 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.692711+0000", 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7021999999999999e-05 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60162 13.2 13:418d4a79:::benchmark_data_smithi042_286876_object60161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.059615904, 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0028493329999999, 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.081 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60162 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865487+0000", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8250000000000001e-06 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865508+0000", 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0947000000000001e-05 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865524+0000", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6052000000000001e-05 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865532+0000", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5989999999999996e-06 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865587+0000", 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5349999999999997e-05 2024-09-13T15:04:37.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865596+0000", 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9719999999999998e-06 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.859408+0000", 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99381218500000001 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860985+0000", 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001576446 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.861455+0000", 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00047016700000000001 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864756+0000", 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003301114 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865824+0000", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010678669999999999 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868257+0000", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024326009999999999 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868292+0000", 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5324000000000003e-05 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868335+0000", 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2883999999999998e-05 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60176 13.b 13:d1c41892:::benchmark_data_smithi042_286876_object60175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.05772080599999, 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99908001400000002, 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.088 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60176 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867381+0000", 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867382+0000", 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5939999999999999e-06 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867408+0000", 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5731000000000001e-05 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867424+0000", 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6475000000000001e-05 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867435+0000", 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0183999999999999e-05 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867533+0000", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8406999999999995e-05 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.867544+0000", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.099e-05 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.858796+0000", 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99125208099999995 2024-09-13T15:04:37.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860588+0000", 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001791988 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860723+0000", 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013452399999999999 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866300+0000", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0055779109999999996 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866355+0000", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4098999999999998e-05 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866368+0000", 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3747999999999999e-05 2024-09-13T15:04:37.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866419+0000", 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0760999999999997e-05 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866461+0000", 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1520999999999998e-05 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60179 13.b 13:da237103:::benchmark_data_smithi042_286876_object60178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.05391090500001, 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.004732119, 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60179 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.096 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871192+0000", 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.708e-06 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871215+0000", 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2867e-05 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871233+0000", 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7872000000000001e-05 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871244+0000", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1304000000000001e-05 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871328+0000", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3912999999999994e-05 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871348+0000", 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9386e-05 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863786+0000", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99243854099999995 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863945+0000", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015913099999999999 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864079+0000", 2024-09-13T15:04:37.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013408800000000001 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872431+0000", 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0083515080000000005 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872656+0000", 2024-09-13T15:04:37.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000225345 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875824+0000", 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031679709999999999 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875879+0000", 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4401000000000003e-05 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875923+0000", 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4084e-05 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60186 13.b 13:db94921c:::benchmark_data_smithi042_286876_object60185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.050402991, 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.001358341, 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.103 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60186 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874700+0000", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.782e-06 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874727+0000", 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6758e-05 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874746+0000", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8569000000000001e-05 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874754+0000", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1769999999999998e-06 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875021+0000", 2024-09-13T15:04:37.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026734699999999997 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875031+0000", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0098999999999999e-05 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866528+0000", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.991496618 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866700+0000", 2024-09-13T15:04:37.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017193800000000001 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866831+0000", 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013130999999999999 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872614+0000", 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057832559999999996 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875962+0000", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003347629 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875974+0000", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2177999999999999e-05 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876022+0000", 2024-09-13T15:04:37.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8270999999999999e-05 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876057+0000", 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4409000000000002e-05 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60204 13.b 13:d5caf6b7:::benchmark_data_smithi042_286876_object60203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:04:37.110 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 184.036025134, 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98712613199999999, 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60204 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.111 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889076+0000", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889079+0000", 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.39e-06 2024-09-13T15:04:37.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889102+0000", 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3379000000000001e-05 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889121+0000", 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8493999999999999e-05 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889131+0000", 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0762e-05 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889195+0000", 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3629999999999999e-05 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.889202+0000", 2024-09-13T15:04:37.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5889999999999997e-06 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868745+0000", 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97954200499999999 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868871+0000", 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012671000000000001 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869018+0000", 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014694099999999999 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872700+0000", 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003681658 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876064+0000", 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033644360000000002 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876097+0000", 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3073e-05 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876158+0000", 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0590000000000001e-05 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876202+0000", 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4474999999999997e-05 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:87021 13.b 13:d10b29d5:::benchmark_data_smithi042_286876_object87020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:55.596179+0000", 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 161.32892276699999, 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.148474515, 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.118 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 87021 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596179+0000", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596179+0000", 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596180+0000", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1990000000000001e-06 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596200+0000", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0492999999999998e-05 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596214+0000", 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3546999999999999e-05 2024-09-13T15:04:37.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596219+0000", 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1739999999999999e-06 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596288+0000", 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8472000000000004e-05 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596386+0000", 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8513999999999999e-05 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.596468+0000", 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2158999999999997e-05 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.602888+0000", 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0064195279999999999 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.602920+0000", 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2400999999999997e-05 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.744381+0000", 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.141461003 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.744608+0000", 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00022709600000000001 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:55.744653+0000", 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4929000000000003e-05 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.124 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89939 13.b 13:d24e6d22:::benchmark_data_smithi042_286876_object89938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.315031+0000", 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.61007071, 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.210419307, 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89939 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.125 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315031+0000", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315031+0000", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315033+0000", 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0789999999999999e-06 2024-09-13T15:04:37.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315055+0000", 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2436e-05 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315068+0000", 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3268000000000001e-05 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315076+0000", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3690000000000001e-06 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315456+0000", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00037995599999999998 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315615+0000", 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000159238 2024-09-13T15:04:37.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.315743+0000", 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000128002 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326376+0000", 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010632727999999999 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326387+0000", 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1059e-05 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525354+0000", 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19896751300000001 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525408+0000", 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3588999999999998e-05 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525450+0000", 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2070000000000002e-05 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:89942 13.3 e359/348 13:cd1c33de:::benchmark_data_smithi042_286876_object89941:head v 359'5584, mlcod=359'5581)", 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.320659+0000", 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.60444285299999, 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20191299200000001, 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.131 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320659+0000", 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320659+0000", 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320660+0000", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.023e-06 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320686+0000", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6811000000000001e-05 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320700+0000", 2024-09-13T15:04:37.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3546999999999999e-05 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320709+0000", 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7630000000000006e-06 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.322111+0000", 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014022100000000001 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.322132+0000", 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1234999999999999e-05 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.329653+0000", 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0075204970000000001 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522572+0000", 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.192918906 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.135 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89948 13.b 13:dff2bd0e:::benchmark_data_smithi042_286876_object89947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.321351+0000", 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.603750316, 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.205128124, 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.136 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89948 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321351+0000", 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321351+0000", 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321353+0000", 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5290000000000001e-06 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321375+0000", 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2549e-05 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321389+0000", 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4003e-05 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.321397+0000", 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7470000000000005e-06 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323040+0000", 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001642985 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323164+0000", 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012407400000000001 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323285+0000", 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000121323 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525320+0000", 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20203495299999999 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525483+0000", 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016306900000000001 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.526392+0000", 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00090907399999999995 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.526442+0000", 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9502999999999997e-05 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.526479+0000", 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7314999999999998e-05 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89951 13.b 13:de3c8d53:::benchmark_data_smithi042_286876_object89950:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:04:37.142 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.323794+0000", 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.60130777800001, 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20467731, 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89951 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323794+0000", 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323794+0000", 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323795+0000", 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2589999999999999e-06 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323821+0000", 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6208e-05 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323833+0000", 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2099999999999999e-05 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323849+0000", 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.543e-05 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326429+0000", 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025807059999999999 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326564+0000", 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013511999999999999 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522065+0000", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19550007699999999 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.526514+0000", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004449789 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.526525+0000", 2024-09-13T15:04:37.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0399e-05 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.528359+0000", 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018345099999999999 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.528430+0000", 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0685999999999994e-05 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.528471+0000", 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1026000000000003e-05 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:89954 13.3 e359/348 13:c3093c51:::benchmark_data_smithi042_286876_object89953:head v 359'5585, mlcod=359'5582)", 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.327786+0000", 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.59731511499999, 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.209497868, 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.149 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327786+0000", 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327786+0000", 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327787+0000", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9800000000000003e-07 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327810+0000", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3192000000000001e-05 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522055+0000", 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.194245116 2024-09-13T15:04:37.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522069+0000", 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3171e-05 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525510+0000", 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034411559999999999 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525526+0000", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6549e-05 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.536181+0000", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010655203 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.537284+0000", 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0011026829999999999 2024-09-13T15:04:37.153 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop_reply(client.45788.0:89939 13.b e359/354 ondisk, result = 0)", 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.329968+0000", 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.59513365699999, 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19548733800000001, 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "started", 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.329968+0000", 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.329968+0000", 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.329969+0000", 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.155e-06 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.329975+0000", 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2060000000000004e-06 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522596+0000", 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.192621394 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522604+0000", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4250000000000001e-06 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525344+0000", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027404059999999999 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525350+0000", 2024-09-13T15:04:37.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4070000000000001e-06 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.525455+0000", 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105345 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_repop(client.45788.0:89966 13.6 e359/344 13:6d88613f:::benchmark_data_smithi042_286876_object89965:head v 359'5605, mlcod=359'5603)", 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.337538+0000", 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 158.587563209, 2024-09-13T15:04:37.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.190296983, 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337538+0000", 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:04:37.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337538+0000", 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337540+0000", 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4330000000000001e-06 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337643+0000", 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010317799999999999 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522728+0000", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18508538399999999 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522736+0000", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5209999999999997e-06 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.522977+0000", 2024-09-13T15:04:37.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00024088899999999999 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523040+0000", 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3083999999999994e-05 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.527789+0000", 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047490950000000001 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:04:37.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.527835+0000", 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6399000000000001e-05 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:04:37.163 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:04:37.163 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:37.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:37.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:37.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.328 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:37.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:37.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:37.467 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:37.569 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:37.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:37.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:37.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:37.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:37.838 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:37.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:37.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:37.973 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:38.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:38.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:37 smithi042 ceph-mon[108570]: pgmap v1393: 99 pgs: 99 active+clean; 930 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:38.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:37 smithi042 ceph-mon[113926]: pgmap v1393: 99 pgs: 99 active+clean; 930 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:38.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:38.136 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:38.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:38.182 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:38.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:38.251 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:37 smithi159 ceph-mon[102519]: pgmap v1393: 99 pgs: 99 active+clean; 930 MiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:38.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:38.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:38.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:38.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:38.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:38.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:38.521 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:38.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:38.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:38.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:38.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:04:38.786 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:38.824 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:38.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:38.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:38.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:38.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:38.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:39.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:39.161 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:39.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:39.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:39.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:39.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.490 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.521 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:39.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:39.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:39.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:39.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:39.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:39.883 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:39] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T15:04:39.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:39.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:39.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:39.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:40.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:40.061 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:40.163 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:40.170 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:39 smithi159 ceph-mon[102519]: pgmap v1394: 99 pgs: 99 active+clean; 839 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:40.200 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:40.202 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:39 smithi042 ceph-mon[108570]: pgmap v1394: 99 pgs: 99 active+clean; 839 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:40.202 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:39 smithi042 ceph-mon[113926]: pgmap v1394: 99 pgs: 99 active+clean; 839 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:40.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:40.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:40.464 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:40.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:40.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.606 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:40.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:40.707 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:40.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:40.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:40.907 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:40.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:40.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:41.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:41.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:41.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.199 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:41.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:41.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:41.328 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:41.430 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:41.455 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:41.520 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:04:41.520 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:04:41.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.520 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:238351 13.d 13:b5882b20:::benchmark_data_smithi042_286876_object114850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.503753+0000", 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.012602788, 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012630583000000001, 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 238351 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503753+0000", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503753+0000", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503752+0000", 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503756+0000", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4490000000000001e-06 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503766+0000", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.73e-06 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503772+0000", 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1510000000000001e-06 2024-09-13T15:04:41.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.507481+0000", 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037086969999999999 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.507843+0000", 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00036278800000000001 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,7", 2024-09-13T15:04:41.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508271+0000", 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00042742600000000001 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516227+0000", 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0079560700000000008 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.525 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:238358 13.e 13:720edee1:::benchmark_data_smithi042_286876_object114857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.508782+0000", 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0075736099999999997, 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0076892360000000003, 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 238358 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508782+0000", 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508782+0000", 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508781+0000", 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508787+0000", 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6710000000000004e-06 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508798+0000", 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1457999999999999e-05 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508806+0000", 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6459999999999993e-06 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508923+0000", 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117061 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.509022+0000", 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9176e-05 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.509118+0000", 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6533999999999994e-05 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.530 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:238353 13.1 e387/344 13:88b935f9:::benchmark_data_smithi042_286876_object114852:head v 387'14937, mlcod=387'14935)", 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.511775+0000", 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0045808740000000004, 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047736799999999998, 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.531 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511775+0000", 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511775+0000", 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511774+0000", 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511781+0000", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8040000000000004e-06 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511793+0000", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2454e-05 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511800+0000", 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7569999999999998e-06 2024-09-13T15:04:41.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511839+0000", 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9561000000000003e-05 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.511859+0000", 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9233999999999999e-05 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:238337 13.5 e387/380 ondisk, result = 0)", 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.515261+0000", 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.001094504, 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013405380000000001, 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515261+0000", 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515261+0000", 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515261+0000", 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515263+0000", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6060000000000001e-06 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515269+0000", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7289999999999997e-06 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515273+0000", 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0609999999999997e-06 2024-09-13T15:04:41.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516246+0000", 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00097234700000000003 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516251+0000", 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0839999999999999e-06 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.538 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:238349 13.5 e387/380)", 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.515277+0000", 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0010789580000000001, 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013804500000000001, 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515277+0000", 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515277+0000", 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515276+0000", 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515278+0000", 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8899999999999999e-06 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515283+0000", 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4859999999999999e-06 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515286+0000", 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4879999999999999e-06 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.541 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:238354 13.1 e387/344 13:8f3d91be:::benchmark_data_smithi042_286876_object114853:head v 387'14938, mlcod=387'14935)", 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.515464+0000", 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00089156599999999999, 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0012410030000000001, 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515464+0000", 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515464+0000", 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515463+0000", 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515469+0000", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4639999999999997e-06 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515476+0000", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2139999999999999e-06 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515481+0000", 2024-09-13T15:04:41.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1109999999999999e-06 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515565+0000", 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3906000000000006e-05 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515583+0000", 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.77e-05 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.545 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:238359 13.6 13:6d4eae6f:::benchmark_data_smithi042_286876_object114858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.515645+0000", 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00071015499999999997, 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0011157070000000001, 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:41.546 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 238359 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515645+0000", 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515645+0000", 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515644+0000", 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515650+0000", 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9619999999999998e-06 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515660+0000", 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.91e-06 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515666+0000", 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.268e-06 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515708+0000", 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2224000000000001e-05 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516016+0000", 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00030782299999999998 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516127+0000", 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011069999999999999 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop_reply(client.45788.0:238358 13.e e387/354)", 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.515978+0000", 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00037792400000000002, 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00085179099999999996, 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.551 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515978+0000", 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515978+0000", 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515977+0000", 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:04:41.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515982+0000", 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.702e-06 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515992+0000", 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0599e-05 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.553 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.515998+0000", 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5979999999999996e-06 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:238337 13.5 13:addb84dd:::benchmark_data_smithi042_286876_object114836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.490528+0000", 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.025827544000000001, 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026359723000000002, 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:41.554 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 238337 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490528+0000", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490528+0000", 2024-09-13T15:04:41.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490526+0000", 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490534+0000", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7319999999999998e-06 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490548+0000", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3599e-05 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.490555+0000", 2024-09-13T15:04:41.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1049999999999997e-06 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499236+0000", 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0086814149999999996 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499440+0000", 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000203329 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.558 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499710+0000", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00027026399999999999 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508690+0000", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0089804029999999996 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.512870+0000", 2024-09-13T15:04:41.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041794509999999998 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516255+0000", 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033854610000000002 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.516828+0000", 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00057270800000000001 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.560 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:238349 13.5 13:ab039d03:::benchmark_data_smithi042_286876_object114848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.499862+0000", 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0164931, 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.017122961999999999, 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:04:41.561 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 238349 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499862+0000", 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499862+0000", 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499861+0000", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499867+0000", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9880000000000001e-06 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499878+0000", 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1293e-05 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.499886+0000", 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6199999999999999e-06 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503093+0000", 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032071629999999999 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:41.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503215+0000", 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121755 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.503331+0000", 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000116338 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.508725+0000", 2024-09-13T15:04:41.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.005393849 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.512877+0000", 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041518850000000001 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 10 2024-09-13T15:04:41.566 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:04:41.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:41.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.611 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:41.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:41.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:41.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:41.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:41.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:41.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:41.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:41.902 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:04:41.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:42.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:42.121 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:42.147 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:41 smithi042 ceph-mon[108570]: pgmap v1395: 99 pgs: 99 active+clean; 804 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T15:04:42.147 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:42.147 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:41 smithi042 ceph-mon[113926]: pgmap v1395: 99 pgs: 99 active+clean; 804 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T15:04:42.147 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:42.154 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:41 smithi159 ceph-mon[102519]: pgmap v1395: 99 pgs: 99 active+clean; 804 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 930 op/s 2024-09-13T15:04:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:04:42.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:42.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:42.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:42.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.427 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:42.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:42.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:04:42.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:42.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:42.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:42.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:42.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:42.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:42.820 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:42.825 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:42.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:42.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:42.978 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:43.009 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:43.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:43.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:42 smithi042 ceph-mon[108570]: osdmap e388: 8 total, 8 up, 7 in 2024-09-13T15:04:43.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:43.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:42 smithi042 ceph-mon[113926]: osdmap e388: 8 total, 8 up, 7 in 2024-09-13T15:04:43.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:43.145 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:43.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/172957959' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:04:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:42 smithi159 ceph-mon[102519]: osdmap e388: 8 total, 8 up, 7 in 2024-09-13T15:04:43.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:43.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:04:43.378 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:43.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:43.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:43.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:43.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.690 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:43.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:43.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:43.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:43.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:43.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:43.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:43.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:43 smithi159 ceph-mon[102519]: pgmap v1397: 99 pgs: 99 active+clean; 589 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:44.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.102 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:44.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.204 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:44.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:44.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:43 smithi042 ceph-mon[108570]: pgmap v1397: 99 pgs: 99 active+clean; 589 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:43 smithi042 ceph-mon[113926]: pgmap v1397: 99 pgs: 99 active+clean; 589 MiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 966 op/s 2024-09-13T15:04:44.376 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:44.385 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:44.487 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:44.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:44.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:44.668 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:44.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:44.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:44.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:44.936 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:44.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:44.951 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:04:45.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:45.075 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:45.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:45.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:45.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:45.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:04:45.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:45.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:45.238 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:04:45.239 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:04:45.239 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:04:45.239 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:04:45.239 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:04:45.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:45.345 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:45.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:45.449 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:45.483 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:45.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:45.585 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:45.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:45.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:45.720 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:45.763 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:45.865 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:45.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:45.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:45.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:45.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:04:46.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:46.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:46.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[108570]: pgmap v1398: 99 pgs: 99 active+clean; 535 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[108570]: osdmap e389: 8 total, 8 up, 7 in 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[113926]: pgmap v1398: 99 pgs: 99 active+clean; 535 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:46.170 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:46.171 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:45 smithi042 ceph-mon[113926]: osdmap e389: 8 total, 8 up, 7 in 2024-09-13T15:04:46.181 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:45 smithi159 ceph-mon[102519]: pgmap v1398: 99 pgs: 99 active+clean; 535 MiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 957 op/s 2024-09-13T15:04:46.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:45 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:04:46.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:45 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2212403438' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:04:46.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:45 smithi159 ceph-mon[102519]: osdmap e389: 8 total, 8 up, 7 in 2024-09-13T15:04:46.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.321 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.333 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:46.423 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:46.459 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:04:46.459 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:46.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:46.482 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:04:46.482 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:46.508 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:04:46.508 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:46.535 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:04:46.535 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:46.568 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:04:46.569 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:46.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:46.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:46.605 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:04:46.606 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:46.635 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:04:46.635 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:46.659 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:04:46.661 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:46.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:46.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:46.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:46.880 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:46.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:46.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:46.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:47.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:47.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:46 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2489642670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:47.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:46 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2489642670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:47.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:47.145 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:47.247 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:47.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:46 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2489642670' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.281 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:47.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:47.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:47.413 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:47.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:47.548 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:47.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:47.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:47.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:47.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:47.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:47.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:47.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:47.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:48.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:48.066 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:47 smithi159 ceph-mon[102519]: pgmap v1400: 99 pgs: 99 active+clean; 335 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:48.088 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:48.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:48.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:48.220 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:48.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:48.322 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:48.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:47 smithi042 ceph-mon[108570]: pgmap v1400: 99 pgs: 99 active+clean; 335 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:48.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:47 smithi042 ceph-mon[113926]: pgmap v1400: 99 pgs: 99 active+clean; 335 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-13T15:04:48.363 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:48.465 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:48.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:48.603 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:48.603 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:48.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:48.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:48.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:48.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:48.870 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:48.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:48.908 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:48.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:49.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:49.047 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:49.149 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:49.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:49.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:49.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:49.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:49.426 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.461 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:49.563 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.596 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:49.650 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:04:49.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:49.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:49.840 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:49.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:49.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:49.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:49.933 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:04:49.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:49 smithi042 ceph-mon[108570]: pgmap v1401: 99 pgs: 99 active+clean; 242 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T15:04:49.937 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:04:49.937 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:49] "GET /metrics HTTP/1.1" 200 36527 "" "Prometheus/2.51.0" 2024-09-13T15:04:49.938 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:49 smithi042 ceph-mon[113926]: pgmap v1401: 99 pgs: 99 active+clean; 242 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T15:04:49.938 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:04:49.969 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:49.979 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:50.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:50.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:50.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:49 smithi159 ceph-mon[102519]: pgmap v1401: 99 pgs: 99 active+clean; 242 MiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s 2024-09-13T15:04:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:04:50.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:50.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:50.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:04:50.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:50.439 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:50.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:50.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:50.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:50.680 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 123498 objects 2024-09-13T15:04:50.680 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :136.675 2024-09-13T15:04:50.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:50.706 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:04:50.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:50.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:50.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:50.831 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:50.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:50.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:50.959 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:50.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:50.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:50.967 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:51.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:51.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:51.185 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:51.207 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:51.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:51.223 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:04:51.223 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:50 smithi159 ceph-mon[102519]: osdmap e390: 8 total, 8 up, 7 in 2024-09-13T15:04:51.223 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.309 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:51.331 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:04:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[108570]: osdmap e390: 8 total, 8 up, 7 in 2024-09-13T15:04:51.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/971629896' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:04:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[113926]: osdmap e390: 8 total, 8 up, 7 in 2024-09-13T15:04:51.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:04:51.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:51.457 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:51.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:51.581 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:51.683 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:51.704 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:51.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:51.806 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:51.828 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:51.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:51.952 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:51.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:51.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:52.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:52.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:52.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:52.198 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:51 smithi159 ceph-mon[102519]: pgmap v1403: 99 pgs: 99 active+clean; 205 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T15:04:52.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:52.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:51 smithi042 ceph-mon[108570]: pgmap v1403: 99 pgs: 99 active+clean; 205 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T15:04:52.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:51 smithi042 ceph-mon[113926]: pgmap v1403: 99 pgs: 99 active+clean; 205 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-13T15:04:52.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:52.553 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:52.577 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:52.635 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:04:52.635 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:04:52.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:52.702 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:52.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:52.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.826 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:52.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:52.928 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:52.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:52.950 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:52.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:53.051 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:53.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:53.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:53.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.198 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:53.299 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:53.365 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:04:53.421 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:53.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.443 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:53.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:53.566 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:53.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:53.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:53.694 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:53.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:53.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:53.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:04:53.819 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:53.921 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:53.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:53.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:53.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:54.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.068 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.077 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:53 smithi042 ceph-mon[113926]: pgmap v1404: 99 pgs: 99 active+clean; 37 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T15:04:54.078 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:53 smithi042 ceph-mon[108570]: pgmap v1404: 99 pgs: 99 active+clean; 37 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T15:04:54.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.192 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:54.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:53 smithi159 ceph-mon[102519]: pgmap v1404: 99 pgs: 99 active+clean; 37 MiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 996 op/s 2024-09-13T15:04:54.294 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:54.316 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:54.326 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:04:54.326 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:04:54.337 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:04:54.337 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:04:54.355 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:04:54.355 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:04:54.367 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:54.374 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:04:54.374 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:04:54.378 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:54.394 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:04:54.394 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:04:54.411 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:04:54.411 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:04:54.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:54.427 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:04:54.428 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:04:54.439 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:54.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:54.444 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:04:54.444 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:04:54.541 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:54.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:54.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.687 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.789 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:54.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:54.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:54.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:54.936 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:54.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:54.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1551099018' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:55.038 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:55.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:55.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:55.184 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:55.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:55.307 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:55.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1551099018' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1551099018' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:04:55.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:55.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:55.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:55.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:55.651 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:04:55.651 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:04:55.651 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:04:55.652 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:04:55.652 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:04:55.660 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:55.682 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:55.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:55.784 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:55.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:55.827 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_8 2024-09-13T15:04:55.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_8 unique_pool_8 --yes-i-really-really-mean-it 2024-09-13T15:04:55.911 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:55.933 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:55.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:56.035 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:56.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:56.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:56.182 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:56.185 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:55 smithi042 ceph-mon[113926]: pgmap v1405: 99 pgs: 99 active+clean; 4.9 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T15:04:56.185 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:55 smithi042 ceph-mon[108570]: pgmap v1405: 99 pgs: 99 active+clean; 4.9 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T15:04:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:55 smithi159 ceph-mon[102519]: pgmap v1405: 99 pgs: 99 active+clean; 4.9 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 869 op/s 2024-09-13T15:04:56.284 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:04:56.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:56.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:04:56.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:56.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:56.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:56.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:56.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:56.776 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:04:56.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:56.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:56.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:56.944 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:56.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:56.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:56.997 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:04:56.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:57.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:57.096 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:57.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:57.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:04:57.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:57.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:57.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:57.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:57.389 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:04:57.447 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:57.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:57.640 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:57.742 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.764 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:57.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:57.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:57.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:57.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:57.958 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:04:57.990 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.016 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.117 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:58.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:58.188 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:57 smithi159 ceph-mon[102519]: pgmap v1406: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2024-09-13T15:04:58.188 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:57 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2555424923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.189 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:57 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.195 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:58.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:58.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[113926]: pgmap v1406: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2555424923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[108570]: pgmap v1406: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2555424923' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:57 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:04:58.364 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:58.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:58.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:58.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.512 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:58.570 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:04:58.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:58.634 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:58.736 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:04:58.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:58.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:04:58.860 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:04:58.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:58.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:58.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:04:58.985 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:04:59.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[108570]: osdmap e391: 8 total, 8 up, 7 in 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:04:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:58 smithi042 ceph-mon[113926]: osdmap e391: 8 total, 8 up, 7 in 2024-09-13T15:04:59.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.133 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:59.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:59.234 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.236 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:58 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:04:59.236 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:58 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:04:59.236 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:58 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:04:59.236 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:58 smithi159 ceph-mon[102519]: osdmap e391: 8 total, 8 up, 7 in 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85746 12.11 12.52b0e931 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 427.83193780900001, 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73855728300000001, 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:04:59.247 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85746 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412238+0000", 2024-09-13T15:04:59.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412236+0000", 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412398+0000", 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000161299 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412410+0000", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2771e-05 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412420+0000", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6369999999999994e-06 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412466+0000", 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5868999999999997e-05 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.412472+0000", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6359999999999997e-06 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150786+0000", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73831398199999998 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150795+0000", 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7150000000000004e-06 2024-09-13T15:04:59.251 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85758 12.11 12.1e200eb1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 427.82185387499999, 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72850151299999999, 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.252 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85758 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422322+0000", 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422320+0000", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422398+0000", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7783999999999999e-05 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422410+0000", 2024-09-13T15:04:59.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1781e-05 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422417+0000", 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5390000000000002e-06 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422459+0000", 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1310000000000003e-05 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.422468+0000", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1190000000000003e-06 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150816+0000", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72834841400000006 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150823+0000", 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0480000000000001e-06 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.256 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85838 12.11 12.30b94631 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 427.74833925500002, 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65501097900000005, 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.257 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85838 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495836+0000", 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495834+0000", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495947+0000", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000113403 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495966+0000", 2024-09-13T15:04:59.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.878e-05 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.495975+0000", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2709999999999998e-06 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.496032+0000", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6221999999999998e-05 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.496043+0000", 2024-09-13T15:04:59.260 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1041e-05 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150840+0000", 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.65479739599999998 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150847+0000", 2024-09-13T15:04:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3220000000000004e-06 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:85847 12.11 12.a88885b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 427.73648995899998, 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64318668999999995, 2024-09-13T15:04:59.263 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 85847 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507686+0000", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507684+0000", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507727+0000", 2024-09-13T15:04:59.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2503000000000003e-05 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507739+0000", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.226e-05 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507745+0000", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8370000000000002e-06 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507795+0000", 2024-09-13T15:04:59.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9666999999999997e-05 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:51.507802+0000", 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9270000000000005e-06 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150865+0000", 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64306261300000001 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.267 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:52.150872+0000", 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3780000000000003e-06 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:91479 12.c 12:3e2fd148:::benchmark_data_smithi042_260592_object91478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 420.00940728900002, 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97834175000000001, 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.268 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 91479 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234768+0000", 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234767+0000", 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234899+0000", 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013134 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234908+0000", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2399999999999996e-06 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234913+0000", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3929999999999999e-06 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234956+0000", 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3319000000000003e-05 2024-09-13T15:04:59.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.234966+0000", 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1079999999999999e-06 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.204917+0000", 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96995188600000004 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.206597+0000", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0016797380000000001 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.206703+0000", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010574899999999999 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.208806+0000", 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002103443 2024-09-13T15:04:59.273 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213022+0000", 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.004215791 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213055+0000", 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3095000000000001e-05 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.213110+0000", 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4880000000000003e-05 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:91482 12.c 12:3b5ee389:::benchmark_data_smithi042_260592_object91481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 420.002618971, 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97600144499999997, 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.275 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 91482 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241557+0000", 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241556+0000", 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241658+0000", 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010255999999999999 2024-09-13T15:04:59.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241671+0000", 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2716e-05 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241678+0000", 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7839999999999998e-06 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241721+0000", 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3451999999999997e-05 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:57:59.241728+0000", 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2069999999999998e-06 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.208867+0000", 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96713846599999997 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.209010+0000", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014311 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.209161+0000", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015071400000000001 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.210792+0000", 2024-09-13T15:04:59.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001631071 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217423+0000", 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066311879999999997 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217470+0000", 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7095999999999998e-05 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:00.217558+0000", 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8199e-05 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151137 12.11 12:8d0cd5bb:::benchmark_data_smithi042_260592_object34246:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.73721051199999, 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0337329420000001, 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.282 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151137 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506965+0000", 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506964+0000", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506970+0000", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.708e-06 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506985+0000", 2024-09-13T15:04:59.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4762e-05 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.506992+0000", 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4200000000000001e-06 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.507029+0000", 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6971999999999998e-05 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.507037+0000", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4170000000000003e-06 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.514264+0000", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0072268550000001 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for missing object", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.514337+0000", 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2965000000000004e-05 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.540613+0000", 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.026276239999999999 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.540698+0000", 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5276000000000001e-05 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151161 12.11 12:8e83c0af:::benchmark_data_smithi042_260592_object34270:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.287 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.70919560499999, 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0002382169999999, 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151161 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:04:59.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534980+0000", 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534978+0000", 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.534986+0000", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9549999999999992e-06 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.535000+0000", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.342e-05 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.535007+0000", 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.99e-06 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.539137+0000", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041303870000000001 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.539150+0000", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3162e-05 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522692+0000", 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98354176100000001 2024-09-13T15:04:59.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522801+0000", 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010924799999999999 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.522871+0000", 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0296000000000006e-05 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530177+0000", 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073050820000000001 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.534953+0000", 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047767649999999997 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535046+0000", 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2238999999999996e-05 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535096+0000", 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0439000000000003e-05 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535218+0000", 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012231000000000001 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.294 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151191 12.11 12:8e8f485d:::benchmark_data_smithi042_260592_object34300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.67140454600002, 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.962574547, 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151191 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:04:59.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572771+0000", 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572769+0000", 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572777+0000", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7249999999999997e-06 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572793+0000", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5471000000000001e-05 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572802+0000", 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1269999999999992e-06 2024-09-13T15:04:59.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572864+0000", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2045999999999995e-05 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.572877+0000", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3001e-05 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524707+0000", 2024-09-13T15:04:59.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.95182974600000003 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524833+0000", 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012646399999999999 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.524963+0000", 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012976099999999999 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.299 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530184+0000", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00522158 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.534996+0000", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048119549999999997 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535254+0000", 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000257781 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.300 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535283+0000", 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8612e-05 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535346+0000", 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2989000000000003e-05 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151243 12.11 12:8d161ebf:::benchmark_data_smithi042_260592_object34352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.301 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.60628711800001, 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90548188600000001, 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151243 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:04:59.302 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637889+0000", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637887+0000", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637894+0000", 2024-09-13T15:04:59.303 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5079999999999999e-06 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637909+0000", 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4885000000000001e-05 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637916+0000", 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4729999999999997e-06 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.304 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637976+0000", 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9852999999999998e-05 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.637998+0000", 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2798000000000001e-05 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526332+0000", 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.88833333299999995 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.305 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526397+0000", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5267999999999999e-05 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.526459+0000", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1657999999999998e-05 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531330+0000", 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0048717489999999999 2024-09-13T15:04:59.306 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535397+0000", 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0040661889999999996 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543256+0000", 2024-09-13T15:04:59.307 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0078591389999999994 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543286+0000", 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0521000000000001e-05 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543370+0000", 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4185999999999997e-05 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.308 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151343 12.11 12:89bd0aa5:::benchmark_data_smithi042_260592_object34452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.51386349299997, 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.81331927999999998, 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151343 2024-09-13T15:04:59.309 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730312+0000", 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.310 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730311+0000", 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730316+0000", 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1849999999999997e-06 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730325+0000", 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5969999999999998e-06 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.311 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730330+0000", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8579999999999999e-06 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730373+0000", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3353999999999999e-05 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.730381+0000", 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1339999999999998e-06 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.312 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527404+0000", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.79702329100000002 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527524+0000", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011975400000000001 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.527642+0000", 2024-09-13T15:04:59.313 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011853000000000001 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531335+0000", 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036927790000000002 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543434+0000", 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012098807 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.314 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543488+0000", 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4348000000000002e-05 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543542+0000", 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3609999999999997e-05 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.543631+0000", 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.9808999999999993e-05 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.315 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151387 12.11 12:8a4c456d:::benchmark_data_smithi042_260592_object34496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.47280315299997, 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.80243400399999998, 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151387 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.316 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771373+0000", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771371+0000", 2024-09-13T15:04:59.317 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771377+0000", 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.054e-06 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771390+0000", 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3179999999999999e-05 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.318 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771397+0000", 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4889999999999998e-06 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:59.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771460+0000", 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2290000000000002e-05 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.771472+0000", 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2118e-05 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.528824+0000", 2024-09-13T15:04:59.321 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.757352001 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.528932+0000", 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107765 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.529042+0000", 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110396 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.322 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533673+0000", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046305799999999996 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.545843+0000", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012170613 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573759+0000", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027915598 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.323 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573784+0000", 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4757999999999999e-05 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573807+0000", 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2994999999999999e-05 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151403 12.11 12:8c51fcd9:::benchmark_data_smithi042_260592_object34512:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:04:59.324 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.45303151799999, 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.78275480399999997, 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151403 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:04:59.325 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791144+0000", 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791142+0000", 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.326 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791148+0000", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.879e-06 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791160+0000", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1532000000000001e-05 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791166+0000", 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2280000000000003e-06 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.327 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791223+0000", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7077000000000003e-05 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.791236+0000", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2486999999999999e-05 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530271+0000", 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.73903509499999998 2024-09-13T15:04:59.328 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530337+0000", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6719999999999998e-05 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.530400+0000", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2718000000000005e-05 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535001+0000", 2024-09-13T15:04:59.329 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0046011639999999996 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560210+0000", 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025208284000000001 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573850+0000", 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013640289 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.330 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573873+0000", 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3402000000000001e-05 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.573899+0000", 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5658000000000001e-05 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151414 12.11 12:8a0a4816:::benchmark_data_smithi042_260592_object34523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.43967668200003, 2024-09-13T15:04:59.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.76962022699999999, 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151414 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804499+0000", 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804497+0000", 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804504+0000", 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.251e-06 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804522+0000", 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8397999999999999e-05 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804529+0000", 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.64e-06 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804587+0000", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8477999999999998e-05 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.804598+0000", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0377000000000001e-05 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531367+0000", 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.72676906399999996 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531474+0000", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107858 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.531596+0000", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012136199999999999 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535006+0000", 2024-09-13T15:04:59.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034101660000000001 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560242+0000", 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025235807999999998 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574034+0000", 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013791954 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574081+0000", 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7638e-05 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574119+0000", 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7851000000000001e-05 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.338 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151428 12.11 12:8a8cc4c2:::benchmark_data_smithi042_260592_object34537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.42622887900001, 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.75627832299999997, 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151428 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817947+0000", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817945+0000", 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817951+0000", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.507e-06 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817962+0000", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1382e-05 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.817969+0000", 2024-09-13T15:04:59.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3910000000000003e-06 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.818009+0000", 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0617000000000001e-05 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.818016+0000", 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5320000000000003e-06 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532552+0000", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.71453657800000003 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532621+0000", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8497000000000005e-05 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.532681+0000", 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.037e-05 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535353+0000", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026720440000000002 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.560772+0000", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025418802000000001 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574179+0000", 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013407211 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574204+0000", 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4708000000000001e-05 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574225+0000", 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1021000000000001e-05 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.345 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151478 12.11 12:898a8b15:::benchmark_data_smithi042_260592_object34587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.37608133999998, 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.70620802299999996, 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151478 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868094+0000", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868093+0000", 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868099+0000", 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6910000000000002e-06 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868110+0000", 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1457999999999999e-05 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868115+0000", 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1399999999999999e-06 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868161+0000", 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5290000000000002e-05 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.868169+0000", 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6200000000000005e-06 2024-09-13T15:04:59.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533705+0000", 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.665535722 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533811+0000", 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106401 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.533921+0000", 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110104 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535483+0000", 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001562 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.569970+0000", 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034486785999999998 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574260+0000", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042902349999999999 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574283+0000", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2860000000000001e-05 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574302+0000", 2024-09-13T15:04:59.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9160999999999998e-05 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151495 12.11 12:88ac8106:::benchmark_data_smithi042_260592_object34604:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.356099809, 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.68633223499999996, 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.353 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151495 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888076+0000", 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888074+0000", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999976 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888082+0000", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1349999999999992e-06 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888098+0000", 2024-09-13T15:04:59.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6269e-05 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888107+0000", 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7010000000000002e-06 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888156+0000", 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8744e-05 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.888162+0000", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8990000000000004e-06 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535524+0000", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64736161699999994 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535635+0000", 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111013 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.535747+0000", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111406 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542042+0000", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0062950209999999996 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570008+0000", 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027966907999999999 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574369+0000", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043603870000000003 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574390+0000", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1605999999999999e-05 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574408+0000", 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.768e-05 2024-09-13T15:04:59.359 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151496 12.11 12:8e7a8881:::benchmark_data_smithi042_260592_object34605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.35303606799999, 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.68339865700000002, 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.360 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151496 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891140+0000", 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891139+0000", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891144+0000", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0329999999999999e-06 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891155+0000", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1548e-05 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891161+0000", 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4709999999999998e-06 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891211+0000", 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9704999999999999e-05 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.891222+0000", 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1589e-05 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.536803+0000", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.64558080200000001 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.536906+0000", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103293 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.537017+0000", 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011120600000000001 2024-09-13T15:04:59.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542046+0000", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0050282490000000003 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570053+0000", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028007695999999999 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574492+0000", 2024-09-13T15:04:59.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0044390619999999997 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574522+0000", 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9238999999999999e-05 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574538+0000", 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6653000000000001e-05 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.366 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151565 12.11 12:8fde0dfc:::benchmark_data_smithi042_260592_object34674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.27068347599999, 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.60117407599999995, 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151565 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.367 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973492+0000", 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973491+0000", 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973497+0000", 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2369999999999997e-06 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973507+0000", 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0118999999999999e-05 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973530+0000", 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2552e-05 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973584+0000", 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4262000000000003e-05 2024-09-13T15:04:59.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.973594+0000", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.628e-06 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538007+0000", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.564413312 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538114+0000", 2024-09-13T15:04:59.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010672599999999999 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.538225+0000", 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011139800000000001 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542049+0000", 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0038240710000000001 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570096+0000", 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028046482000000001 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574609+0000", 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045130530000000004 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574635+0000", 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5803999999999999e-05 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574666+0000", 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1739999999999998e-05 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:151571 12.11 12:8fdec748:::benchmark_data_smithi042_260592_object34680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 363.26329284000002, 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.59386337899999997, 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:04:59.374 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 151571 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980883+0000", 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980882+0000", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980887+0000", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4550000000000003e-06 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980898+0000", 2024-09-13T15:04:59.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1250000000000001e-05 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980904+0000", 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4450000000000004e-06 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980946+0000", 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2249999999999997e-05 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:04:59.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:55.980953+0000", 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.294e-06 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539191+0000", 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.55823776700000005 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539298+0000", 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000106983 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.378 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.539406+0000", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107598 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.542053+0000", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026469359999999999 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.570127+0000", 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028074492 2024-09-13T15:04:59.379 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574701+0000", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045736379999999997 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574728+0000", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7042999999999998e-05 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:58:56.574746+0000", 2024-09-13T15:04:59.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8464000000000001e-05 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:04:59.381 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:04:59.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:59.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:59.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:59.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:04:59.567 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:59.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:04:59.669 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:04:59.691 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:59.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:04:59.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:04:59.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:04:59.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:04:59.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:04:59.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:04:59.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:04:59.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:04:59.960 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:04:59.991 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[108570]: pgmap v1408: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 441 op/s 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2207499374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[113926]: pgmap v1408: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 441 op/s 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2207499374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:04:59.992 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:04:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:04:59] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-09-13T15:05:00.041 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:00.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:00.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:00.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:00.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:59 smithi159 ceph-mon[102519]: pgmap v1408: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 441 op/s 2024-09-13T15:05:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2207499374' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:05:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:04:59 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:05:00.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:00.312 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:00.413 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:00.536 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:00.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:00.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:00.573 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:05:00.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:00.680 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:00.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:00.782 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:00.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:00.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:00.927 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:00.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:01.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.050 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:05:01.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:00 smithi042 ceph-mon[108570]: osdmap e392: 8 total, 8 up, 7 in 2024-09-13T15:05:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:05:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:00 smithi042 ceph-mon[113926]: osdmap e392: 8 total, 8 up, 7 in 2024-09-13T15:05:01.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:05:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:00 smithi159 ceph-mon[102519]: osdmap e392: 8 total, 8 up, 7 in 2024-09-13T15:05:01.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:01.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:01.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:01.435 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:05:01.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:01.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:01.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:01.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:01.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:01.793 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:01.877 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:01.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:01.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:01.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:01.971 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_8' removed 2024-09-13T15:05:02.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:02.026 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:01 smithi159 ceph-mon[102519]: pgmap v1410: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T15:05:02.026 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:05:02.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:02.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:02.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:02.180 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:02.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:02.249 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:01 smithi042 ceph-mon[108570]: pgmap v1410: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T15:05:02.249 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:05:02.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:01 smithi042 ceph-mon[113926]: pgmap v1410: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 71 op/s 2024-09-13T15:05:02.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:05:02.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:02.296 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:02.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:02.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:02.407 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:02.428 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:02.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:02.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:02.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:02.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.679 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:02.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:02.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:02.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:02.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:02.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:02.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:02.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:03.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:03.053 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:03.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:03.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:05:03.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:02 smithi159 ceph-mon[102519]: osdmap e393: 8 total, 8 up, 7 in 2024-09-13T15:05:03.256 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:05:03.257 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:02 smithi042 ceph-mon[113926]: osdmap e393: 8 total, 8 up, 7 in 2024-09-13T15:05:03.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2371785671' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_8", "pool2": "unique_pool_8", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:05:03.257 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:02 smithi042 ceph-mon[108570]: osdmap e393: 8 total, 8 up, 7 in 2024-09-13T15:05:03.278 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-13T15:05:03.279 INFO:tasks.radosbench:Beginning radosbench... 2024-09-13T15:05:03.280 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_9 2024-09-13T15:05:03.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_9 16 2024-09-13T15:05:03.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:03.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:03.404 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:03.429 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:03.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:03.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:03.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:03.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:03.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:03.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:03.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:03.736 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:05:03.737 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:03.738 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:03.738 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:03.778 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:03.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:03.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:03.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:03.924 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:03.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:04.026 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:04.047 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:04.149 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:04.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:03 smithi159 ceph-mon[102519]: pgmap v1412: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2372936424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:04.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:04.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:03 smithi042 ceph-mon[108570]: pgmap v1412: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:04.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2372936424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:04.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:03 smithi042 ceph-mon[113926]: pgmap v1412: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:04.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2372936424' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:04.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:04.416 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:04.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:04.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:04.644 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:04.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.667 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:04.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:04.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:04.841 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:04.841 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:04.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:04.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:04.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:04.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:05.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:05.042 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:05.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:05.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.167 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:05.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:05.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:05.293 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:05.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:05.395 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:05.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:05.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:05.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:05:05.464 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:05:05.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:05.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:05.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:05.665 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:05.767 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:05.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:05.890 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:05.911 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:05.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:05.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:05 smithi159 ceph-mon[102519]: pgmap v1413: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:06.013 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:06.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[113926]: pgmap v1413: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:06.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[108570]: pgmap v1413: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:06.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:06.138 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:06.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:06.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:06.228 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:06.261 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:06.283 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:06.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:06.407 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:06.492 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:06.494 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:05:06.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:06.532 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:06.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:06.633 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.655 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:06.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:06.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:06.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:06.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:06.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:07.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:07.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:07.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:07.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:07.154 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:07.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:07.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:07.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:07.381 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:07.403 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:07.505 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:07.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:07.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:07.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:07.655 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:07.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:07.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.781 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:07.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:07.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:07.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:07.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:07.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:08.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:08.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:08.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:08.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:07 smithi159 ceph-mon[102519]: pgmap v1414: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:08.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:07 smithi042 ceph-mon[108570]: pgmap v1414: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:08.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:07 smithi042 ceph-mon[113926]: pgmap v1414: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:08.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:05:08.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.396 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:08.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:08.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:08.566 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:08.566 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:08.585 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:05:08.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.643 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.644 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:08.771 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:08.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:08.873 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:08.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:08.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:08.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:08.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:09.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:09.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:09.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:09.120 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[113926]: osdmap e394: 8 total, 8 up, 7 in 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[108570]: osdmap e394: 8 total, 8 up, 7 in 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.121 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.142 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:09.212 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:08 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:09.212 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/878540746' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:05:09.212 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:08 smithi159 ceph-mon[102519]: osdmap e394: 8 total, 8 up, 7 in 2024-09-13T15:05:09.212 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.212 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:09.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:05:09.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:09.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:09.399 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:09.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:09.433 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:09.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:09.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:09.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:09.590 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:09.613 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:09.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:09.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:09.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:09.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:09.897 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:09.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:09.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:09.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:09.968 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:09] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.51.0" 2024-09-13T15:05:10.020 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:10.041 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:10.095 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_9' created 2024-09-13T15:05:10.143 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:10.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:10.270 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:10.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:10.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[108570]: pgmap v1416: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1715359293' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[113926]: pgmap v1416: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1715359293' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:10 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.395 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:10.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:10.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:10 smithi159 ceph-mon[102519]: pgmap v1416: 83 pgs: 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:10 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1715359293' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:10 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16} : dispatch 2024-09-13T15:05:10.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:10.540 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:10.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:10.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:10.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:10.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:10.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:10.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:10.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:10.913 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:10.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:11.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:11.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:05:11.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:11.170 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:11 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-13T15:05:11.170 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:11 smithi159 ceph-mon[102519]: osdmap e395: 8 total, 8 up, 7 in 2024-09-13T15:05:11.173 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:11.173 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:11.189 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:11.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:11.312 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:11.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:11 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-13T15:05:11.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:11 smithi042 ceph-mon[108570]: osdmap e395: 8 total, 8 up, 7 in 2024-09-13T15:05:11.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:11 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_9", "pg_num": 16}]': finished 2024-09-13T15:05:11.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:11 smithi042 ceph-mon[113926]: osdmap e395: 8 total, 8 up, 7 in 2024-09-13T15:05:11.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:11.436 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:11.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:11.538 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:11.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:11.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.685 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:11.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:11.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.808 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:11.874 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:05:11.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:11.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:11.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:11.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:12.035 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:12.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:12.072 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:12.158 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:12.182 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:12.193 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:12 smithi159 ceph-mon[102519]: pgmap v1418: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:12.193 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:12 smithi159 ceph-mon[102519]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:05:12.193 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:12 smithi159 ceph-mon[102519]: osdmap e396: 8 total, 8 up, 7 in 2024-09-13T15:05:12.284 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:12.305 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[108570]: pgmap v1418: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[108570]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:05:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[108570]: osdmap e396: 8 total, 8 up, 7 in 2024-09-13T15:05:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[113926]: pgmap v1418: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[113926]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:05:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:12 smithi042 ceph-mon[113926]: osdmap e396: 8 total, 8 up, 7 in 2024-09-13T15:05:12.408 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:12.429 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:12.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:12.553 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:12.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:12.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:12.804 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:12.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_9 rados --yes-i-really-mean-it || true 2024-09-13T15:05:12.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:12.929 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:12.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:13.031 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:13.053 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:13.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:13.155 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:13.177 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:13.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:13.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:13 smithi159 ceph-mon[102519]: osdmap e397: 8 total, 8 up, 7 in 2024-09-13T15:05:13.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:13.300 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:13.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:13.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:13.424 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:13.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:13.547 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:13.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:13 smithi042 ceph-mon[108570]: osdmap e397: 8 total, 8 up, 7 in 2024-09-13T15:05:13.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:13 smithi042 ceph-mon[113926]: osdmap e397: 8 total, 8 up, 7 in 2024-09-13T15:05:13.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:13.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:13.755 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:13.755 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:13.773 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:13.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:13.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:13.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:13.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:13.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:14.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:14.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:14.146 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:14.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:14.231 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:14 smithi042 ceph-mon[113926]: pgmap v1421: 99 pgs: 2 creating+activating, 5 unknown, 92 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:14.231 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:14 smithi042 ceph-mon[108570]: pgmap v1421: 99 pgs: 2 creating+activating, 5 unknown, 92 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:14.270 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:14.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:14.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:14.333 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:05:14.333 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:14.346 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:05:14.346 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:14.363 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:05:14.363 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:14.381 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:05:14.381 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:14.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:14.398 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:05:14.398 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:14.414 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:05:14.414 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.415 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:14.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:14.431 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:05:14.431 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:14.448 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:05:14.448 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:14.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:14 smithi159 ceph-mon[102519]: pgmap v1421: 99 pgs: 2 creating+activating, 5 unknown, 92 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:14.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:05:14.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:14.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:14.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:14.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:14.695 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:14.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:14.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:14.889 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:14.911 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:14.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:14.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:15.013 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:15.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:15.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:15.110 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:05:15.136 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:15.158 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:15.197 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2588948080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:15.198 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2275866869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.198 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.198 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2588948080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:15.198 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2275866869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.198 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.259 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:15.282 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:15.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:15.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:15.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:15.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2588948080' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:15.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2275866869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:05:15.507 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:15.529 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:15.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.654 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:15.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:15.756 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:15.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:15.873 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:05:15.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:05:15.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:15.902 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:15.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:15.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:16.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:16.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:16.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:16.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:16.184 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:16 smithi159 ceph-mon[102519]: pgmap v1422: 99 pgs: 2 creating+activating, 2 unknown, 95 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:16.184 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:16 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:05:16.184 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:16 smithi159 ceph-mon[102519]: osdmap e398: 8 total, 8 up, 7 in 2024-09-13T15:05:16.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:16.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:16.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.399 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:16.447 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:05:16.447 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:05:16.448 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:05:16.448 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:05:16.448 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:05:16.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:16.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[108570]: pgmap v1422: 99 pgs: 2 creating+activating, 2 unknown, 95 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:05:16.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[108570]: osdmap e398: 8 total, 8 up, 7 in 2024-09-13T15:05:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[113926]: pgmap v1422: 99 pgs: 2 creating+activating, 2 unknown, 95 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:05:16.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:16 smithi042 ceph-mon[113926]: osdmap e398: 8 total, 8 up, 7 in 2024-09-13T15:05:16.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:16.637 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:16.651 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:16.681 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:16.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:16.775 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:16.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:16.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:16.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:16.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.066 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:05:17.124 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:17.146 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:17.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:17.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:17.266 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:05:17.270 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:17.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:17.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:17.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:17.495 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:17.517 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:17.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:17.641 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:17.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.765 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:17.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:17.867 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:17.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:18.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:18.119 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:18.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:18.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:18.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:18.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:18.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:18.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:18.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:18.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:18.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:18.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:18 smithi159 ceph-mon[102519]: pgmap v1424: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:18.512 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:18 smithi042 ceph-mon[113926]: pgmap v1424: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:18.513 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:18 smithi042 ceph-mon[108570]: pgmap v1424: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:18.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:18.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:18.623 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:18.645 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:18.747 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:18.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:18.871 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:18.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:18.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:18.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:19.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:19.117 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:19.140 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:19.242 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:19.265 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:19.329 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:05:19.329 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:224630 11.0 e293/244 11:01630af9:::benchmark_data_smithi042_223320_object105624:head v 293'14176, mlcod=293'14173)", 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:55:45.100508+0000", 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 574.22596888299995, 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21657052500000001, 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.330 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.100508+0000", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.100508+0000", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.100507+0000", 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.331 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.100521+0000", 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4847e-05 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308892+0000", 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.20837049299999999 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.332 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308908+0000", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6208000000000001e-05 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.309026+0000", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0001176 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.309048+0000", 2024-09-13T15:05:19.333 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1934e-05 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.317048+0000", 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0080006629999999999 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.317078+0000", 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9709e-05 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.334 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45314.0:224639 11.5 e293/244 11:ae494322:::benchmark_data_smithi042_223320_object105633:head v 293'13774, mlcod=293'13772)", 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T14:55:45.107105+0000", 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 574.21937146799996, 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.21583295899999999, 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.335 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.107105+0000", 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.107105+0000", 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.107104+0000", 2024-09-13T15:05:19.336 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.107113+0000", 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6409999999999997e-06 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.337 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308890+0000", 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.20177610600000001 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308908+0000", 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8440999999999998e-05 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.338 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308959+0000", 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0699999999999999e-05 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.308982+0000", 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3723999999999999e-05 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.322895+0000", 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.013912677 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.339 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T14:55:45.322938+0000", 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2874000000000003e-05 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:222639 12.13 e337/315 12:cd4bf2cd:::benchmark_data_smithi042_260592_object105748:head v 337'9570, mlcod=337'9568)", 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:08.710302+0000", 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 310.616174908, 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.083656328000000002, 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.340 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710302+0000", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710302+0000", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.341 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710301+0000", 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710309+0000", 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9270000000000005e-06 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710318+0000", 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0310000000000004e-06 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.342 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710324+0000", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.891e-06 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710373+0000", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9802999999999997e-05 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.710394+0000", 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1004000000000001e-05 2024-09-13T15:05:19.343 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.793898+0000", 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.083503588000000004 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.793958+0000", 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9874000000000003e-05 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.344 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:222640 12.3 e337/315 12:c131fb06:::benchmark_data_smithi042_260592_object105749:head v 337'9479, mlcod=337'9477)", 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:08.711909+0000", 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 310.61456745700002, 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.082076990000000002, 2024-09-13T15:05:19.345 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711909+0000", 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.346 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711909+0000", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711908+0000", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711916+0000", 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5259999999999996e-06 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.347 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711926+0000", 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0083e-05 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711932+0000", 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3010000000000003e-06 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.348 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.711983+0000", 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0705999999999999e-05 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712006+0000", 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2605000000000001e-05 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.793961+0000", 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.081955754000000006 2024-09-13T15:05:19.349 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.793986+0000", 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.4660000000000001e-05 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:222644 12.13 e337/315 12:cf2d1d3a:::benchmark_data_smithi042_260592_object105753:head v 337'9571, mlcod=337'9568)", 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:08.712656+0000", 2024-09-13T15:05:19.350 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 310.61382023900001, 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.081352414999999997, 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712656+0000", 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712656+0000", 2024-09-13T15:05:19.351 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712656+0000", 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712662+0000", 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2079999999999997e-06 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.352 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712672+0000", 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8730000000000002e-06 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.712677+0000", 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7479999999999999e-06 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.353 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713377+0000", 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00069954299999999995 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713439+0000", 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2082999999999995e-05 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.793988+0000", 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.080548700000000001 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.354 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.794009+0000", 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0947000000000001e-05 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:222649 12.e e337/326 12:7383f6b1:::benchmark_data_smithi042_260592_object105758:head v 337'13846, mlcod=337'13844)", 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:08.713721+0000", 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 310.61275538000001, 2024-09-13T15:05:19.355 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.092713200999999995, 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713721+0000", 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713721+0000", 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.356 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713720+0000", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713727+0000", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0709999999999997e-06 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713738+0000", 2024-09-13T15:05:19.357 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0225e-05 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713744+0000", 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5010000000000001e-06 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.713784+0000", 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9375999999999997e-05 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.358 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.790862+0000", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.077078811999999997 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.806359+0000", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.015497039000000001 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.806434+0000", 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4892000000000001e-05 2024-09-13T15:05:19.359 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45599.0:222646 12.17 e337/307 12:efd09cb3:::benchmark_data_smithi042_260592_object105755:head v 337'9264, mlcod=337'9262)", 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:08.714405+0000", 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 310.61207167200001, 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.082579484999999994, 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.360 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.714405+0000", 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.714405+0000", 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.714404+0000", 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.361 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.714410+0000", 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0349999999999998e-06 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.791070+0000", 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.076660323000000002 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.362 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.791079+0000", 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3520000000000007e-06 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.791121+0000", 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2289000000000001e-05 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.791143+0000", 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1747999999999999e-05 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.363 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.796907+0000", 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057640349999999998 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.796984+0000", 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7491999999999996e-05 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.364 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110697 13.f e361/344 13:f677430e:::benchmark_data_smithi042_286876_object110696:head v 361'6864, mlcod=361'6862)", 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 179.754894401, 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33391684900000002, 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.365 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571581+0000", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571627+0000", 2024-09-13T15:05:19.366 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5974999999999999e-05 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571640+0000", 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2938e-05 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571647+0000", 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7179999999999999e-06 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.367 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571691+0000", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4731999999999998e-05 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571711+0000", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0064e-05 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905463+0000", 2024-09-13T15:05:19.368 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33375128300000001 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905499+0000", 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6229000000000002e-05 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110698 13.d e361/344 13:b5e3a503:::benchmark_data_smithi042_286876_object110697:head v 361'6993, mlcod=361'6991)", 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 179.754186659, 2024-09-13T15:05:19.369 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34348563700000001, 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.370 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572289+0000", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572332+0000", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2815000000000003e-05 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572343+0000", 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1163e-05 2024-09-13T15:05:19.371 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572349+0000", 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3430000000000001e-06 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572404+0000", 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4199000000000001e-05 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.372 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572422+0000", 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8601999999999999e-05 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.915732+0000", 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34331013199999999 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.915775+0000", 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3176999999999997e-05 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.373 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110704 13.e e361/354 13:793880cd:::benchmark_data_smithi042_286876_object110703:head v 361'6942, mlcod=361'6940)", 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 179.752238205, 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33492928500000002, 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.374 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.375 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574237+0000", 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574388+0000", 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000150798 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905431+0000", 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33104323499999999 2024-09-13T15:05:19.376 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905444+0000", 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2891000000000001e-05 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905490+0000", 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.613e-05 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.377 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905526+0000", 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5531000000000003e-05 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.909089+0000", 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035635139999999998 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.909168+0000", 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8368000000000006e-05 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.378 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:135784 13.f e366/344 13:fb444c05:::benchmark_data_smithi042_286876_object12283:head v 366'8466, mlcod=366'8463)", 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 152.60209169300001, 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040515675000000001, 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.379 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724384+0000", 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.380 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724390+0000", 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6509999999999998e-06 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724402+0000", 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1885999999999999e-05 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.381 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724408+0000", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2790000000000004e-06 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.730074+0000", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056665070000000003 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.730093+0000", 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8859999999999999e-05 2024-09-13T15:05:19.382 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.764858+0000", 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034764694999999998 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.764900+0000", 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2410000000000002e-05 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.383 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237925 13.7 e387/344 13:e4e8a23a:::benchmark_data_smithi042_286876_object114424:head v 387'14553, mlcod=387'14549)", 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.260785671000001, 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.041768151000000003, 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.384 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065690+0000", 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.385 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065696+0000", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4470000000000003e-06 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065713+0000", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6371e-05 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065720+0000", 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9550000000000003e-06 2024-09-13T15:05:19.386 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066779+0000", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0010590269999999999 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066835+0000", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.647e-05 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107422+0000", 2024-09-13T15:05:19.387 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040587217000000002 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107459+0000", 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6520999999999998e-05 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237928 13.7 e387/344 13:e0e58808:::benchmark_data_smithi042_286876_object114427:head v 387'14554, mlcod=387'14550)", 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.259981154999998, 2024-09-13T15:05:19.388 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040986935000000002, 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:05:19.389 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.390 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066501+0000", 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9719999999999997e-06 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066524+0000", 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3447000000000001e-05 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066531+0000", 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1810000000000004e-06 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.391 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.071239+0000", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047075370000000004 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.071304+0000", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5053000000000002e-05 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107465+0000", 2024-09-13T15:05:19.392 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.036161445 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107482+0000", 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7028999999999999e-05 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237932 13.f e387/344 13:f7f5141f:::benchmark_data_smithi042_286876_object114431:head v 387'14815, mlcod=387'14813)", 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:05:19.393 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.258200872000003, 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.039224614999999997, 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.394 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068275+0000", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068281+0000", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5810000000000002e-06 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068296+0000", 2024-09-13T15:05:19.395 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4861000000000001e-05 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068303+0000", 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.579e-06 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.072526+0000", 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042231179999999997 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.396 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.072584+0000", 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7729000000000002e-05 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107484+0000", 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034900333999999998 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107500+0000", 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6285000000000001e-05 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.397 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237933 13.f e387/344 13:f3c23bc3:::benchmark_data_smithi042_286876_object114432:head v 387'14816, mlcod=387'14813)", 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.249005189000002, 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.039545807000000002, 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.398 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077470+0000", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077477+0000", 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7059999999999998e-06 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.399 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077487+0000", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0474e-05 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077493+0000", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1489999999999999e-06 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079962+0000", 2024-09-13T15:05:19.400 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024686460000000001 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.080007+0000", 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5399999999999999e-05 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.111523+0000", 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.031515504 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.401 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117017+0000", 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0054940689999999999 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237934 13.f e387/344 13:f0a0c5a8:::benchmark_data_smithi042_286876_object114433:head v 387'14817, mlcod=387'14813)", 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.247270323999999, 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037943330999999997, 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.402 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.403 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079205+0000", 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079212+0000", 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8920000000000003e-06 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079458+0000", 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00024603299999999999 2024-09-13T15:05:19.404 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079465+0000", 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1790000000000002e-06 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.087686+0000", 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.008220158 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.405 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.087763+0000", 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7601000000000005e-05 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117024+0000", 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.029260699000000001 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.407 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117149+0000", 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012558900000000001 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237942 13.7 e387/344 13:eb3314b0:::benchmark_data_smithi042_286876_object114441:head v 387'14555, mlcod=387'14551)", 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.247004273999998, 2024-09-13T15:05:19.408 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037819001999999997, 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:05:19.409 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079476+0000", 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7060000000000003e-06 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.410 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079483+0000", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4760000000000003e-06 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079487+0000", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9659999999999998e-06 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.091989+0000", 2024-09-13T15:05:19.411 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012501945 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.092056+0000", 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6649999999999994e-05 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117155+0000", 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025099082000000002 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.412 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117291+0000", 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000136549 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237950 13.7 e387/344 13:ee7c64af:::benchmark_data_smithi042_286876_object114449:head v 387'14556, mlcod=387'14551)", 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 38.242274021, 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037050540999999999, 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.413 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:05:19.414 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084208+0000", 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.891e-06 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084218+0000", 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0497e-05 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.415 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084224+0000", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9510000000000003e-06 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.103717+0000", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019492651999999999 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.103736+0000", 2024-09-13T15:05:19.416 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9004e-05 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.121185+0000", 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.017449479 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.121253+0000", 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7773000000000003e-05 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.417 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:242882 13.f e389/344 13:fe304d65:::benchmark_data_smithi042_286876_object119381:head v 389'15121, mlcod=389'15119)", 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 33.001046021999997, 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.045855901999999997, 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.418 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:05:19.419 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325436+0000", 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3350000000000003e-06 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325450+0000", 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3821e-05 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.420 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325456+0000", 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5350000000000001e-06 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.327766+0000", 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023101490000000001 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.327836+0000", 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9504000000000004e-05 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.421 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371244+0000", 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.043408081000000001 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371286+0000", 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2271999999999998e-05 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:242886 13.f e389/344 13:fed0232b:::benchmark_data_smithi042_286876_object119385:head v 389'15122, mlcod=389'15119)", 2024-09-13T15:05:19.422 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 32.995691897999997, 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040524537999999999, 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:05:19.423 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330784+0000", 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330790+0000", 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3239999999999999e-06 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.424 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330802+0000", 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1375999999999999e-05 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330808+0000", 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1249999999999998e-06 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.425 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.334018+0000", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032106410000000002 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.334041+0000", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2461000000000001e-05 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371291+0000", 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037250162000000003 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:05:19.426 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371309+0000", 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8139999999999999e-05 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:05:19.427 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:05:19.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:19.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:19.552 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:19.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:19.654 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:19.677 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:19.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:19.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:19.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:19] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.51.0" 2024-09-13T15:05:19.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:19.926 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:19.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:20.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:20.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:20.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:20.176 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:20.188 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:05:20.237 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:20 smithi159 ceph-mon[102519]: pgmap v1425: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:20.237 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1206852528' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.237 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.237 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.237 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:05:20.279 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:20.300 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:20.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:20.402 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:20.430 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:20.508 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[113926]: pgmap v1425: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:20.508 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1206852528' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.508 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.508 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[108570]: pgmap v1425: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1206852528' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.509 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:20.533 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:20.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:20.658 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:20.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:20.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:20.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:20.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:20.931 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:20.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:21.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:21.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:21.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:21.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:21.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:21.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:21.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.303 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:21.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:21.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:21.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: osdmap e399: 8 total, 8 up, 7 in 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:21.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: osdmap e399: 8 total, 8 up, 7 in 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.537 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: osdmap e399: 8 total, 8 up, 7 in 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:21.538 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:21.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:21.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:21.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:21.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:21.779 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:21.800 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:21.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:21.902 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:21.923 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:21.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:21.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:22.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:22.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:22.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:22.147 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.171 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:22.273 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:22.294 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:22.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:22.395 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:22.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:22 smithi159 ceph-mon[102519]: pgmap v1427: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:22 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:05:22.520 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:22 smithi042 ceph-mon[108570]: pgmap v1427: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:22.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:22 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:05:22.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:22 smithi042 ceph-mon[113926]: pgmap v1427: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:22.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:22 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-13T15:05:22.538 INFO:teuthology.orchestra.run.smithi042.stderr:enabled application 'rados' on pool 'unique_pool_9' 2024-09-13T15:05:22.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:22.649 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:22.671 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:22.773 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:22.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:22.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:22.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:22.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:22.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:23.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:23.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:23.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:23.172 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:23.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:23.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:23.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:23.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:23.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:23.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:23.525 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.531 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:23.551 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:23.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.574 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:23.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[108570]: osdmap e400: 8 total, 8 up, 7 in 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[108570]: pgmap v1429: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[113926]: osdmap e400: 8 total, 8 up, 7 in 2024-09-13T15:05:23.657 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:23 smithi042 ceph-mon[113926]: pgmap v1429: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:23.678 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:23.780 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:23.803 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:23.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:23.905 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:23.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:23.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:23.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:23 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1717327184' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_9", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-13T15:05:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:23 smithi159 ceph-mon[102519]: osdmap e400: 8 total, 8 up, 7 in 2024-09-13T15:05:23.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:23 smithi159 ceph-mon[102519]: pgmap v1429: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:24.032 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:24.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:24.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:24.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:05:24.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:24.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:24.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:24.310 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:24.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.311 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:24.406 DEBUG:teuthology.orchestra.run.smithi042:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_9 bench 90 write' 2024-09-13T15:05:24.408 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-13T15:05:24.408 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-13T15:05:24.412 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:24.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:24.460 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:hints = 1 2024-09-13T15:05:24.460 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-13T15:05:24.460 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object prefix: benchmark_data_smithi042_319156 2024-09-13T15:05:24.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:05:24.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 0 0 0 0 0 0 - 0 2024-09-13T15:05:24.538 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:24.570 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:24.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:24.712 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:24.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:24.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:24.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:24 smithi042 ceph-mon[108570]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:05:24.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:24 smithi042 ceph-mon[113926]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:05:24.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:24.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:24.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:24.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:24.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:24.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:24 smithi159 ceph-mon[102519]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T15:05:25.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:25.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:25.223 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:25.254 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:25.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:25.355 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:25.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:25.460 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 1 16 3833 3817 238.549 238.562 0.00435809 0.00417776 2024-09-13T15:05:25.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:25.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:25.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:25.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:25.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:25.755 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:25 smithi042 ceph-mon[113926]: pgmap v1430: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:25.755 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:25 smithi042 ceph-mon[108570]: pgmap v1430: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:25.765 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:25.807 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:25.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:25.911 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:05:25.914 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:05:25.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:25.960 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:25.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:25.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:25.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:25 smithi159 ceph-mon[102519]: pgmap v1430: 99 pgs: 99 active+clean; 582 KiB data, 7.7 GiB used, 618 GiB / 626 GiB avail 2024-09-13T15:05:26.062 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:26.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:26.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:26.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:26.246 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:26.348 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:26.386 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:26.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 2 16 6955 6939 216.828 195.125 0.00557807 0.00460262 2024-09-13T15:05:26.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:26.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:26.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.662 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:26.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:26.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:26.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:26.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:26 smithi042 ceph-mon[113926]: from='client.46130 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:26.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:26 smithi042 ceph-mon[108570]: from='client.46130 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:26.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:26.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:26.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:26.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:26 smithi159 ceph-mon[102519]: from='client.46130 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:27.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:27.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:27.175 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:27.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:27.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 3 16 9925 9909 206.421 185.625 0.00309376 0.00483531 2024-09-13T15:05:27.475 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.577 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:27.603 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:27.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:27.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:27.769 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:05:27.769 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:27.809 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:27 smithi042 ceph-mon[113926]: pgmap v1431: 99 pgs: 99 active+clean; 208 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 26 MiB/s wr, 414 op/s 2024-09-13T15:05:27.809 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:27 smithi042 ceph-mon[108570]: pgmap v1431: 99 pgs: 99 active+clean; 208 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 26 MiB/s wr, 414 op/s 2024-09-13T15:05:27.810 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:05:27.810 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:27.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:27.847 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:05:27.847 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:27.880 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:27.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:27.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:27.886 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:05:27.886 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:27.920 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:05:27.921 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:27.957 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:05:27.957 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:27.982 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:27.991 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:05:27.991 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:27.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:27 smithi159 ceph-mon[102519]: pgmap v1431: 99 pgs: 99 active+clean; 208 MiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 26 MiB/s wr, 414 op/s 2024-09-13T15:05:28.025 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:05:28.026 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.027 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:28.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:28.128 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:28.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:28.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:28.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:28.297 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:28.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:28.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:28.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:28.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 4 16 12587 12571 196.406 166.375 0.00875718 0.00508196 2024-09-13T15:05:28.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:28.579 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:28.681 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:28.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:28.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:28.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2491711476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:28.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2491711476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:28.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:28.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:28.996 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:28.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:28.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2491711476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:29.037 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:29.045 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1430,"stamp":"2024-09-13T15:05:24.711052+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8103308,"kb_used_data":37752,"kb_used_omap":22,"kb_used_meta":8065449,"kb_avail":648112756,"statfs":{"total":671965249536,"available":663667462144,"internally_reserved":0,"allocated":38658048,"data_stored":23573921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":8259020007},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001764"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":625,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636031+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:05:20.636031+0000","last_peered":"2024-09-13T15:05:20.636031+0000","last_clean":"2024-09-13T15:05:20.636031+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:05:20.636031+0000","last_undegraded":"2024-09-13T15:05:20.636031+0000","last_fullsized":"2024-09-13T15:05:20.636031+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":92,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107785+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:05:15.107785+0000","last_peered":"2024-09-13T15:05:15.107785+0000","last_clean":"2024-09-13T15:05:15.107785+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:05:15.107785+0000","last_undegraded":"2024-09-13T15:05:15.107785+0000","last_fullsized":"2024-09-13T15:05:15.107785+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636014+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:05:20.636014+0000","last_peered":"2024-09-13T15:05:20.636014+0000","last_clean":"2024-09-13T15:05:20.636014+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:05:20.636014+0000","last_undegraded":"2024-09-13T15:05:20.636014+0000","last_fullsized":"2024-09-13T15:05:20.636014+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":394,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635356+0000","last_change":"2024-09-13T15:03:44.123745+0000","last_active":"2024-09-13T15:05:20.635356+0000","last_peered":"2024-09-13T15:05:20.635356+0000","last_clean":"2024-09-13T15:05:20.635356+0000","last_became_active":"2024-09-13T15:03:44.123397+0000","last_became_peered":"2024-09-13T15:03:44.123397+0000","last_unstale":"2024-09-13T15:05:20.635356+0000","last_undegraded":"2024-09-13T15:05:20.635356+0000","last_fullsized":"2024-09-13T15:05:20.635356+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.079957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":395,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635497+0000","last_change":"2024-09-13T15:03:44.128445+0000","last_active":"2024-09-13T15:05:20.635497+0000","last_peered":"2024-09-13T15:05:20.635497+0000","last_clean":"2024-09-13T15:05:20.635497+0000","last_became_active":"2024-09-13T15:03:44.128047+0000","last_became_peered":"2024-09-13T15:03:44.128047+0000","last_unstale":"2024-09-13T15:05:20.635497+0000","last_undegraded":"2024-09-13T15:05:20.635497+0000","last_fullsized":"2024-09-13T15:05:20.635497+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:22.359631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107381+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:05:15.107381+0000","last_peered":"2024-09-13T15:05:15.107381+0000","last_clean":"2024-09-13T15:05:15.107381+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:05:15.107381+0000","last_undegraded":"2024-09-13T15:05:15.107381+0000","last_fullsized":"2024-09-13T15:05:15.107381+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":651,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636407+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:05:20.636407+0000","last_peered":"2024-09-13T15:05:20.636407+0000","last_clean":"2024-09-13T15:05:20.636407+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:05:20.636407+0000","last_undegraded":"2024-09-13T15:05:20.636407+0000","last_fullsized":"2024-09-13T15:05:20.636407+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":492,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286465+0000","last_change":"2024-09-13T15:03:44.130129+0000","last_active":"2024-09-13T15:05:16.286465+0000","last_peered":"2024-09-13T15:05:16.286465+0000","last_clean":"2024-09-13T15:05:16.286465+0000","last_became_active":"2024-09-13T15:03:44.114883+0000","last_became_peered":"2024-09-13T15:03:44.114883+0000","last_unstale":"2024-09-13T15:05:16.286465+0000","last_undegraded":"2024-09-13T15:05:16.286465+0000","last_fullsized":"2024-09-13T15:05:16.286465+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:54.544220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286389+0000","last_change":"2024-09-13T15:05:11.104668+0000","last_active":"2024-09-13T15:05:16.286389+0000","last_peered":"2024-09-13T15:05:16.286389+0000","last_clean":"2024-09-13T15:05:16.286389+0000","last_became_active":"2024-09-13T15:05:11.104349+0000","last_became_peered":"2024-09-13T15:05:11.104349+0000","last_unstale":"2024-09-13T15:05:16.286389+0000","last_undegraded":"2024-09-13T15:05:16.286389+0000","last_fullsized":"2024-09-13T15:05:16.286389+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:34.717367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":114,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108022+0000","last_change":"2024-09-13T15:03:44.129600+0000","last_active":"2024-09-13T15:05:15.108022+0000","last_peered":"2024-09-13T15:05:15.108022+0000","last_clean":"2024-09-13T15:05:15.108022+0000","last_became_active":"2024-09-13T15:03:44.123858+0000","last_became_peered":"2024-09-13T15:03:44.123858+0000","last_unstale":"2024-09-13T15:05:15.108022+0000","last_undegraded":"2024-09-13T15:05:15.108022+0000","last_fullsized":"2024-09-13T15:05:15.108022+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:15.345841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":57,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286359+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:05:16.286359+0000","last_peered":"2024-09-13T15:05:16.286359+0000","last_clean":"2024-09-13T15:05:16.286359+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:05:16.286359+0000","last_undegraded":"2024-09-13T15:05:16.286359+0000","last_fullsized":"2024-09-13T15:05:16.286359+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"0'0","reported_seq":18,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636102+0000","last_change":"2024-09-13T15:05:11.103932+0000","last_active":"2024-09-13T15:05:20.636102+0000","last_peered":"2024-09-13T15:05:20.636102+0000","last_clean":"2024-09-13T15:05:20.636102+0000","last_became_active":"2024-09-13T15:05:11.103484+0000","last_became_peered":"2024-09-13T15:05:11.103484+0000","last_unstale":"2024-09-13T15:05:20.636102+0000","last_undegraded":"2024-09-13T15:05:20.636102+0000","last_fullsized":"2024-09-13T15:05:20.636102+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:25:52.798601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":90,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108029+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T15:05:15.108029+0000","last_peered":"2024-09-13T15:05:15.108029+0000","last_clean":"2024-09-13T15:05:15.108029+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T15:05:15.108029+0000","last_undegraded":"2024-09-13T15:05:15.108029+0000","last_fullsized":"2024-09-13T15:05:15.108029+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:20.897906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107564+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T15:05:15.107564+0000","last_peered":"2024-09-13T15:05:15.107564+0000","last_clean":"2024-09-13T15:05:15.107564+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T15:05:15.107564+0000","last_undegraded":"2024-09-13T15:05:15.107564+0000","last_fullsized":"2024-09-13T15:05:15.107564+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"0'0","reported_seq":21,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538541+0000","last_change":"2024-09-13T15:05:11.537086+0000","last_active":"2024-09-13T15:05:22.538541+0000","last_peered":"2024-09-13T15:05:22.538541+0000","last_clean":"2024-09-13T15:05:22.538541+0000","last_became_active":"2024-09-13T15:05:11.536730+0000","last_became_peered":"2024-09-13T15:05:11.536730+0000","last_unstale":"2024-09-13T15:05:22.538541+0000","last_undegraded":"2024-09-13T15:05:22.538541+0000","last_fullsized":"2024-09-13T15:05:22.538541+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:51.566063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":69,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286680+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:05:16.286680+0000","last_peered":"2024-09-13T15:05:16.286680+0000","last_clean":"2024-09-13T15:05:16.286680+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:05:16.286680+0000","last_undegraded":"2024-09-13T15:05:16.286680+0000","last_fullsized":"2024-09-13T15:05:16.286680+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":374,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286501+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T15:05:16.286501+0000","last_peered":"2024-09-13T15:05:16.286501+0000","last_clean":"2024-09-13T15:05:16.286501+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T15:05:16.286501+0000","last_undegraded":"2024-09-13T15:05:16.286501+0000","last_fullsized":"2024-09-13T15:05:16.286501+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"0'0","reported_seq":18,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636089+0000","last_change":"2024-09-13T15:05:11.101187+0000","last_active":"2024-09-13T15:05:20.636089+0000","last_peered":"2024-09-13T15:05:20.636089+0000","last_clean":"2024-09-13T15:05:20.636089+0000","last_became_active":"2024-09-13T15:05:11.100618+0000","last_became_peered":"2024-09-13T15:05:11.100618+0000","last_unstale":"2024-09-13T15:05:20.636089+0000","last_undegraded":"2024-09-13T15:05:20.636089+0000","last_fullsized":"2024-09-13T15:05:20.636089+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:20:25.083443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":190,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636168+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:05:20.636168+0000","last_peered":"2024-09-13T15:05:20.636168+0000","last_clean":"2024-09-13T15:05:20.636168+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:05:20.636168+0000","last_undegraded":"2024-09-13T15:05:20.636168+0000","last_fullsized":"2024-09-13T15:05:20.636168+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":426,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109367+0000","last_change":"2024-09-13T15:03:44.122648+0000","last_active":"2024-09-13T15:05:15.109367+0000","last_peered":"2024-09-13T15:05:15.109367+0000","last_clean":"2024-09-13T15:05:15.109367+0000","last_became_active":"2024-09-13T15:03:44.121071+0000","last_became_peered":"2024-09-13T15:03:44.121071+0000","last_unstale":"2024-09-13T15:05:15.109367+0000","last_undegraded":"2024-09-13T15:05:15.109367+0000","last_fullsized":"2024-09-13T15:05:15.109367+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:51.385950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"0'0","reported_seq":21,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539197+0000","last_change":"2024-09-13T15:05:11.103812+0000","last_active":"2024-09-13T15:05:22.539197+0000","last_peered":"2024-09-13T15:05:22.539197+0000","last_clean":"2024-09-13T15:05:22.539197+0000","last_became_active":"2024-09-13T15:05:11.103500+0000","last_became_peered":"2024-09-13T15:05:11.103500+0000","last_unstale":"2024-09-13T15:05:22.539197+0000","last_undegraded":"2024-09-13T15:05:22.539197+0000","last_fullsized":"2024-09-13T15:05:22.539197+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:33.535670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":977,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635730+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:05:20.635730+0000","last_peered":"2024-09-13T15:05:20.635730+0000","last_clean":"2024-09-13T15:05:20.635730+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:05:20.635730+0000","last_undegraded":"2024-09-13T15:05:20.635730+0000","last_fullsized":"2024-09-13T15:05:20.635730+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":700,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635853+0000","last_change":"2024-09-13T15:03:44.133692+0000","last_active":"2024-09-13T15:05:20.635853+0000","last_peered":"2024-09-13T15:05:20.635853+0000","last_clean":"2024-09-13T15:05:20.635853+0000","last_became_active":"2024-09-13T15:03:44.125117+0000","last_became_peered":"2024-09-13T15:03:44.125117+0000","last_unstale":"2024-09-13T15:05:20.635853+0000","last_undegraded":"2024-09-13T15:05:20.635853+0000","last_fullsized":"2024-09-13T15:05:20.635853+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:12.857175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107778+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:05:15.107778+0000","last_peered":"2024-09-13T15:05:15.107778+0000","last_clean":"2024-09-13T15:05:15.107778+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:05:15.107778+0000","last_undegraded":"2024-09-13T15:05:15.107778+0000","last_fullsized":"2024-09-13T15:05:15.107778+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107953+0000","last_change":"2024-09-13T15:05:11.537076+0000","last_active":"2024-09-13T15:05:15.107953+0000","last_peered":"2024-09-13T15:05:15.107953+0000","last_clean":"2024-09-13T15:05:15.107953+0000","last_became_active":"2024-09-13T15:05:11.536775+0000","last_became_peered":"2024-09-13T15:05:11.536775+0000","last_unstale":"2024-09-13T15:05:15.107953+0000","last_undegraded":"2024-09-13T15:05:15.107953+0000","last_fullsized":"2024-09-13T15:05:15.107953+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:38.213878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":476,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538781+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:05:22.538781+0000","last_peered":"2024-09-13T15:05:22.538781+0000","last_clean":"2024-09-13T15:05:22.538781+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:05:22.538781+0000","last_undegraded":"2024-09-13T15:05:22.538781+0000","last_fullsized":"2024-09-13T15:05:22.538781+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":422,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286639+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T15:05:16.286639+0000","last_peered":"2024-09-13T15:05:16.286639+0000","last_clean":"2024-09-13T15:05:16.286639+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T15:05:16.286639+0000","last_undegraded":"2024-09-13T15:05:16.286639+0000","last_fullsized":"2024-09-13T15:05:16.286639+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107891+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T15:05:15.107891+0000","last_peered":"2024-09-13T15:05:15.107891+0000","last_clean":"2024-09-13T15:05:15.107891+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T15:05:15.107891+0000","last_undegraded":"2024-09-13T15:05:15.107891+0000","last_fullsized":"2024-09-13T15:05:15.107891+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.287009+0000","last_change":"2024-09-13T15:05:11.105322+0000","last_active":"2024-09-13T15:05:16.287009+0000","last_peered":"2024-09-13T15:05:16.287009+0000","last_clean":"2024-09-13T15:05:16.287009+0000","last_became_active":"2024-09-13T15:05:11.104691+0000","last_became_peered":"2024-09-13T15:05:11.104691+0000","last_unstale":"2024-09-13T15:05:16.287009+0000","last_undegraded":"2024-09-13T15:05:16.287009+0000","last_fullsized":"2024-09-13T15:05:16.287009+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:23.376418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107449+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:05:15.107449+0000","last_peered":"2024-09-13T15:05:15.107449+0000","last_clean":"2024-09-13T15:05:15.107449+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:05:15.107449+0000","last_undegraded":"2024-09-13T15:05:15.107449+0000","last_fullsized":"2024-09-13T15:05:15.107449+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108066+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T15:05:15.108066+0000","last_peered":"2024-09-13T15:05:15.108066+0000","last_clean":"2024-09-13T15:05:15.108066+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T15:05:15.108066+0000","last_undegraded":"2024-09-13T15:05:15.108066+0000","last_fullsized":"2024-09-13T15:05:15.108066+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107749+0000","last_change":"2024-09-13T15:05:11.103992+0000","last_active":"2024-09-13T15:05:15.107749+0000","last_peered":"2024-09-13T15:05:15.107749+0000","last_clean":"2024-09-13T15:05:15.107749+0000","last_became_active":"2024-09-13T15:05:11.103694+0000","last_became_peered":"2024-09-13T15:05:11.103694+0000","last_unstale":"2024-09-13T15:05:15.107749+0000","last_undegraded":"2024-09-13T15:05:15.107749+0000","last_fullsized":"2024-09-13T15:05:15.107749+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:15.405086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":476,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539176+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:05:22.539176+0000","last_peered":"2024-09-13T15:05:22.539176+0000","last_clean":"2024-09-13T15:05:22.539176+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:05:22.539176+0000","last_undegraded":"2024-09-13T15:05:22.539176+0000","last_fullsized":"2024-09-13T15:05:22.539176+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":628,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636001+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:05:20.636001+0000","last_peered":"2024-09-13T15:05:20.636001+0000","last_clean":"2024-09-13T15:05:20.636001+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:05:20.636001+0000","last_undegraded":"2024-09-13T15:05:20.636001+0000","last_fullsized":"2024-09-13T15:05:20.636001+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":639,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109411+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:05:15.109411+0000","last_peered":"2024-09-13T15:05:15.109411+0000","last_clean":"2024-09-13T15:05:15.109411+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:05:15.109411+0000","last_undegraded":"2024-09-13T15:05:15.109411+0000","last_fullsized":"2024-09-13T15:05:15.109411+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"0'0","reported_seq":18,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636324+0000","last_change":"2024-09-13T15:05:11.105485+0000","last_active":"2024-09-13T15:05:20.636324+0000","last_peered":"2024-09-13T15:05:20.636324+0000","last_clean":"2024-09-13T15:05:20.636324+0000","last_became_active":"2024-09-13T15:05:11.105170+0000","last_became_peered":"2024-09-13T15:05:11.105170+0000","last_unstale":"2024-09-13T15:05:20.636324+0000","last_undegraded":"2024-09-13T15:05:20.636324+0000","last_fullsized":"2024-09-13T15:05:20.636324+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:44:22.010494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":598,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286879+0000","last_change":"2024-09-13T15:03:46.574157+0000","last_active":"2024-09-13T15:05:16.286879+0000","last_peered":"2024-09-13T15:05:16.286879+0000","last_clean":"2024-09-13T15:05:16.286879+0000","last_became_active":"2024-09-13T15:03:46.572750+0000","last_became_peered":"2024-09-13T15:03:46.572750+0000","last_unstale":"2024-09-13T15:05:16.286879+0000","last_undegraded":"2024-09-13T15:05:16.286879+0000","last_fullsized":"2024-09-13T15:05:16.286879+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:55.892848+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":114,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538672+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:05:22.538672+0000","last_peered":"2024-09-13T15:05:22.538672+0000","last_clean":"2024-09-13T15:05:22.538672+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:05:22.538672+0000","last_undegraded":"2024-09-13T15:05:22.538672+0000","last_fullsized":"2024-09-13T15:05:22.538672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":613,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635734+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:05:20.635734+0000","last_peered":"2024-09-13T15:05:20.635734+0000","last_clean":"2024-09-13T15:05:20.635734+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:05:20.635734+0000","last_undegraded":"2024-09-13T15:05:20.635734+0000","last_fullsized":"2024-09-13T15:05:20.635734+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":384,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286676+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:05:16.286676+0000","last_peered":"2024-09-13T15:05:16.286676+0000","last_clean":"2024-09-13T15:05:16.286676+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:05:16.286676+0000","last_undegraded":"2024-09-13T15:05:16.286676+0000","last_fullsized":"2024-09-13T15:05:16.286676+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107663+0000","last_change":"2024-09-13T15:05:11.104165+0000","last_active":"2024-09-13T15:05:15.107663+0000","last_peered":"2024-09-13T15:05:15.107663+0000","last_clean":"2024-09-13T15:05:15.107663+0000","last_became_active":"2024-09-13T15:05:11.103885+0000","last_became_peered":"2024-09-13T15:05:11.103885+0000","last_unstale":"2024-09-13T15:05:15.107663+0000","last_undegraded":"2024-09-13T15:05:15.107663+0000","last_fullsized":"2024-09-13T15:05:15.107663+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:32.640246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":906,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109322+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:05:15.109322+0000","last_peered":"2024-09-13T15:05:15.109322+0000","last_clean":"2024-09-13T15:05:15.109322+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:05:15.109322+0000","last_undegraded":"2024-09-13T15:05:15.109322+0000","last_fullsized":"2024-09-13T15:05:15.109322+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":630,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108792+0000","last_change":"2024-09-13T15:03:44.117533+0000","last_active":"2024-09-13T15:05:15.108792+0000","last_peered":"2024-09-13T15:05:15.108792+0000","last_clean":"2024-09-13T15:05:15.108792+0000","last_became_active":"2024-09-13T15:03:44.117216+0000","last_became_peered":"2024-09-13T15:03:44.117216+0000","last_unstale":"2024-09-13T15:05:15.108792+0000","last_undegraded":"2024-09-13T15:05:15.108792+0000","last_fullsized":"2024-09-13T15:05:15.108792+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:17.222690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":190,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107306+0000","last_change":"2024-09-13T15:03:44.115491+0000","last_active":"2024-09-13T15:05:15.107306+0000","last_peered":"2024-09-13T15:05:15.107306+0000","last_clean":"2024-09-13T15:05:15.107306+0000","last_became_active":"2024-09-13T15:03:44.114978+0000","last_became_peered":"2024-09-13T15:03:44.114978+0000","last_unstale":"2024-09-13T15:05:15.107306+0000","last_undegraded":"2024-09-13T15:05:15.107306+0000","last_fullsized":"2024-09-13T15:05:15.107306+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:04.996891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":464,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108608+0000","last_change":"2024-09-13T15:03:44.117741+0000","last_active":"2024-09-13T15:05:15.108608+0000","last_peered":"2024-09-13T15:05:15.108608+0000","last_clean":"2024-09-13T15:05:15.108608+0000","last_became_active":"2024-09-13T15:03:44.117404+0000","last_became_peered":"2024-09-13T15:03:44.117404+0000","last_unstale":"2024-09-13T15:05:15.108608+0000","last_undegraded":"2024-09-13T15:05:15.108608+0000","last_fullsized":"2024-09-13T15:05:15.108608+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:02.169360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286596+0000","last_change":"2024-09-13T15:05:11.537213+0000","last_active":"2024-09-13T15:05:16.286596+0000","last_peered":"2024-09-13T15:05:16.286596+0000","last_clean":"2024-09-13T15:05:16.286596+0000","last_became_active":"2024-09-13T15:05:11.536909+0000","last_became_peered":"2024-09-13T15:05:11.536909+0000","last_unstale":"2024-09-13T15:05:16.286596+0000","last_undegraded":"2024-09-13T15:05:16.286596+0000","last_fullsized":"2024-09-13T15:05:16.286596+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:13.844350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":646,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107405+0000","last_change":"2024-09-13T15:03:46.571609+0000","last_active":"2024-09-13T15:05:15.107405+0000","last_peered":"2024-09-13T15:05:15.107405+0000","last_clean":"2024-09-13T15:05:15.107405+0000","last_became_active":"2024-09-13T15:03:46.571111+0000","last_became_peered":"2024-09-13T15:03:46.571111+0000","last_unstale":"2024-09-13T15:05:15.107405+0000","last_undegraded":"2024-09-13T15:05:15.107405+0000","last_fullsized":"2024-09-13T15:05:15.107405+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:26.923160+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":686,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538587+0000","last_change":"2024-09-13T15:03:44.113114+0000","last_active":"2024-09-13T15:05:22.538587+0000","last_peered":"2024-09-13T15:05:22.538587+0000","last_clean":"2024-09-13T15:05:22.538587+0000","last_became_active":"2024-09-13T15:03:44.112798+0000","last_became_peered":"2024-09-13T15:03:44.112798+0000","last_unstale":"2024-09-13T15:05:22.538587+0000","last_undegraded":"2024-09-13T15:05:22.538587+0000","last_fullsized":"2024-09-13T15:05:22.538587+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:39.690606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":624,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108967+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:05:15.108967+0000","last_peered":"2024-09-13T15:05:15.108967+0000","last_clean":"2024-09-13T15:05:15.108967+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:05:15.108967+0000","last_undegraded":"2024-09-13T15:05:15.108967+0000","last_fullsized":"2024-09-13T15:05:15.108967+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":692,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109274+0000","last_change":"2024-09-13T15:03:44.120103+0000","last_active":"2024-09-13T15:05:15.109274+0000","last_peered":"2024-09-13T15:05:15.109274+0000","last_clean":"2024-09-13T15:05:15.109274+0000","last_became_active":"2024-09-13T15:03:44.119715+0000","last_became_peered":"2024-09-13T15:03:44.119715+0000","last_unstale":"2024-09-13T15:05:15.109274+0000","last_undegraded":"2024-09-13T15:05:15.109274+0000","last_fullsized":"2024-09-13T15:05:15.109274+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:16.484960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"0'0","reported_seq":13,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107939+0000","last_change":"2024-09-13T15:05:11.104779+0000","last_active":"2024-09-13T15:05:15.107939+0000","last_peered":"2024-09-13T15:05:15.107939+0000","last_clean":"2024-09-13T15:05:15.107939+0000","last_became_active":"2024-09-13T15:05:11.104444+0000","last_became_peered":"2024-09-13T15:05:11.104444+0000","last_unstale":"2024-09-13T15:05:15.107939+0000","last_undegraded":"2024-09-13T15:05:15.107939+0000","last_fullsized":"2024-09-13T15:05:15.107939+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:23.452855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":439,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538299+0000","last_change":"2024-09-13T15:03:47.227943+0000","last_active":"2024-09-13T15:05:22.538299+0000","last_peered":"2024-09-13T15:05:22.538299+0000","last_clean":"2024-09-13T15:05:22.538299+0000","last_became_active":"2024-09-13T15:03:46.585701+0000","last_became_peered":"2024-09-13T15:03:46.585701+0000","last_unstale":"2024-09-13T15:05:22.538299+0000","last_undegraded":"2024-09-13T15:05:22.538299+0000","last_fullsized":"2024-09-13T15:05:22.538299+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:03:47.226199+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:03:47.226199+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:12.460412+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":115,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109366+0000","last_change":"2024-09-13T15:01:33.852433+0000","last_active":"2024-09-13T15:05:15.109366+0000","last_peered":"2024-09-13T15:05:15.109366+0000","last_clean":"2024-09-13T15:05:15.109366+0000","last_became_active":"2024-09-13T15:01:33.852015+0000","last_became_peered":"2024-09-13T15:01:33.852015+0000","last_unstale":"2024-09-13T15:05:15.109366+0000","last_undegraded":"2024-09-13T15:05:15.109366+0000","last_fullsized":"2024-09-13T15:05:15.109366+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:52:03.246939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":106,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107700+0000","last_change":"2024-09-13T15:03:44.120307+0000","last_active":"2024-09-13T15:05:15.107700+0000","last_peered":"2024-09-13T15:05:15.107700+0000","last_clean":"2024-09-13T15:05:15.107700+0000","last_became_active":"2024-09-13T15:03:44.119949+0000","last_became_peered":"2024-09-13T15:03:44.119949+0000","last_unstale":"2024-09-13T15:05:15.107700+0000","last_undegraded":"2024-09-13T15:05:15.107700+0000","last_fullsized":"2024-09-13T15:05:15.107700+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.569133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":637,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109228+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:05:15.109228+0000","last_peered":"2024-09-13T15:05:15.109228+0000","last_clean":"2024-09-13T15:05:15.109228+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:05:15.109228+0000","last_undegraded":"2024-09-13T15:05:15.109228+0000","last_fullsized":"2024-09-13T15:05:15.109228+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"0'0","reported_seq":16,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108550+0000","last_change":"2024-09-13T15:05:11.105474+0000","last_active":"2024-09-13T15:05:15.108550+0000","last_peered":"2024-09-13T15:05:15.108550+0000","last_clean":"2024-09-13T15:05:15.108550+0000","last_became_active":"2024-09-13T15:05:11.105182+0000","last_became_peered":"2024-09-13T15:05:11.105182+0000","last_unstale":"2024-09-13T15:05:15.108550+0000","last_undegraded":"2024-09-13T15:05:15.108550+0000","last_fullsized":"2024-09-13T15:05:15.108550+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:42:05.759654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":651,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286938+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:05:16.286938+0000","last_peered":"2024-09-13T15:05:16.286938+0000","last_clean":"2024-09-13T15:05:16.286938+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:05:16.286938+0000","last_undegraded":"2024-09-13T15:05:16.286938+0000","last_fullsized":"2024-09-13T15:05:16.286938+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636202+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:05:20.636202+0000","last_peered":"2024-09-13T15:05:20.636202+0000","last_clean":"2024-09-13T15:05:20.636202+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:05:20.636202+0000","last_undegraded":"2024-09-13T15:05:20.636202+0000","last_fullsized":"2024-09-13T15:05:20.636202+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":707,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109465+0000","last_change":"2024-09-13T15:03:44.116244+0000","last_active":"2024-09-13T15:05:15.109465+0000","last_peered":"2024-09-13T15:05:15.109465+0000","last_clean":"2024-09-13T15:05:15.109465+0000","last_became_active":"2024-09-13T15:03:44.115926+0000","last_became_peered":"2024-09-13T15:03:44.115926+0000","last_unstale":"2024-09-13T15:05:15.109465+0000","last_undegraded":"2024-09-13T15:05:15.109465+0000","last_fullsized":"2024-09-13T15:05:15.109465+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:01.231884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":563,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109115+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:05:15.109115+0000","last_peered":"2024-09-13T15:05:15.109115+0000","last_clean":"2024-09-13T15:05:15.109115+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:05:15.109115+0000","last_undegraded":"2024-09-13T15:05:15.109115+0000","last_fullsized":"2024-09-13T15:05:15.109115+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.9","version":"0'0","reported_seq":18,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635621+0000","last_change":"2024-09-13T15:05:11.537327+0000","last_active":"2024-09-13T15:05:20.635621+0000","last_peered":"2024-09-13T15:05:20.635621+0000","last_clean":"2024-09-13T15:05:20.635621+0000","last_became_active":"2024-09-13T15:05:11.536882+0000","last_became_peered":"2024-09-13T15:05:11.536882+0000","last_unstale":"2024-09-13T15:05:20.635621+0000","last_undegraded":"2024-09-13T15:05:20.635621+0000","last_fullsized":"2024-09-13T15:05:20.635621+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:55:32.480164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":717,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286847+0000","last_change":"2024-09-13T15:03:46.574886+0000","last_active":"2024-09-13T15:05:16.286847+0000","last_peered":"2024-09-13T15:05:16.286847+0000","last_clean":"2024-09-13T15:05:16.286847+0000","last_became_active":"2024-09-13T15:03:46.574563+0000","last_became_peered":"2024-09-13T15:03:46.574563+0000","last_unstale":"2024-09-13T15:05:16.286847+0000","last_undegraded":"2024-09-13T15:05:16.286847+0000","last_fullsized":"2024-09-13T15:05:16.286847+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:29.856598+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":624,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109459+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:05:15.109459+0000","last_peered":"2024-09-13T15:05:15.109459+0000","last_clean":"2024-09-13T15:05:15.109459+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:05:15.109459+0000","last_undegraded":"2024-09-13T15:05:15.109459+0000","last_fullsized":"2024-09-13T15:05:15.109459+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":190,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108044+0000","last_change":"2024-09-13T15:03:44.121114+0000","last_active":"2024-09-13T15:05:15.108044+0000","last_peered":"2024-09-13T15:05:15.108044+0000","last_clean":"2024-09-13T15:05:15.108044+0000","last_became_active":"2024-09-13T15:03:44.120758+0000","last_became_peered":"2024-09-13T15:03:44.120758+0000","last_unstale":"2024-09-13T15:05:15.108044+0000","last_undegraded":"2024-09-13T15:05:15.108044+0000","last_fullsized":"2024-09-13T15:05:15.108044+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:54.923156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":62,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109268+0000","last_change":"2024-09-13T15:03:44.114482+0000","last_active":"2024-09-13T15:05:15.109268+0000","last_peered":"2024-09-13T15:05:15.109268+0000","last_clean":"2024-09-13T15:05:15.109268+0000","last_became_active":"2024-09-13T15:03:44.112887+0000","last_became_peered":"2024-09-13T15:03:44.112887+0000","last_unstale":"2024-09-13T15:05:15.109268+0000","last_undegraded":"2024-09-13T15:05:15.109268+0000","last_fullsized":"2024-09-13T15:05:15.109268+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:37.719076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.a","version":"0'0","reported_seq":16,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109022+0000","last_change":"2024-09-13T15:05:11.537005+0000","last_active":"2024-09-13T15:05:15.109022+0000","last_peered":"2024-09-13T15:05:15.109022+0000","last_clean":"2024-09-13T15:05:15.109022+0000","last_became_active":"2024-09-13T15:05:11.536634+0000","last_became_peered":"2024-09-13T15:05:11.536634+0000","last_unstale":"2024-09-13T15:05:15.109022+0000","last_undegraded":"2024-09-13T15:05:15.109022+0000","last_fullsized":"2024-09-13T15:05:15.109022+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:02:41.336210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":579,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107971+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:05:15.107971+0000","last_peered":"2024-09-13T15:05:15.107971+0000","last_clean":"2024-09-13T15:05:15.107971+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:05:15.107971+0000","last_undegraded":"2024-09-13T15:05:15.107971+0000","last_fullsized":"2024-09-13T15:05:15.107971+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":658,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636480+0000","last_change":"2024-09-13T15:03:44.131836+0000","last_active":"2024-09-13T15:05:20.636480+0000","last_peered":"2024-09-13T15:05:20.636480+0000","last_clean":"2024-09-13T15:05:20.636480+0000","last_became_active":"2024-09-13T15:03:44.120633+0000","last_became_peered":"2024-09-13T15:03:44.120633+0000","last_unstale":"2024-09-13T15:05:20.636480+0000","last_undegraded":"2024-09-13T15:05:20.636480+0000","last_fullsized":"2024-09-13T15:05:20.636480+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:52.065604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":638,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108946+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:05:15.108946+0000","last_peered":"2024-09-13T15:05:15.108946+0000","last_clean":"2024-09-13T15:05:15.108946+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:05:15.108946+0000","last_undegraded":"2024-09-13T15:05:15.108946+0000","last_fullsized":"2024-09-13T15:05:15.108946+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":658,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635486+0000","last_change":"2024-09-13T15:03:44.123687+0000","last_active":"2024-09-13T15:05:20.635486+0000","last_peered":"2024-09-13T15:05:20.635486+0000","last_clean":"2024-09-13T15:05:20.635486+0000","last_became_active":"2024-09-13T15:03:44.123135+0000","last_became_peered":"2024-09-13T15:03:44.123135+0000","last_unstale":"2024-09-13T15:05:20.635486+0000","last_undegraded":"2024-09-13T15:05:20.635486+0000","last_fullsized":"2024-09-13T15:05:20.635486+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:42.101656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"0'0","reported_seq":21,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538431+0000","last_change":"2024-09-13T15:05:11.104176+0000","last_active":"2024-09-13T15:05:22.538431+0000","last_peered":"2024-09-13T15:05:22.538431+0000","last_clean":"2024-09-13T15:05:22.538431+0000","last_became_active":"2024-09-13T15:05:11.103879+0000","last_became_peered":"2024-09-13T15:05:11.103879+0000","last_unstale":"2024-09-13T15:05:22.538431+0000","last_undegraded":"2024-09-13T15:05:22.538431+0000","last_fullsized":"2024-09-13T15:05:22.538431+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:43:06.183789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":887,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635923+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:05:20.635923+0000","last_peered":"2024-09-13T15:05:20.635923+0000","last_clean":"2024-09-13T15:05:20.635923+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:05:20.635923+0000","last_undegraded":"2024-09-13T15:05:20.635923+0000","last_fullsized":"2024-09-13T15:05:20.635923+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":417,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108700+0000","last_change":"2024-09-13T15:03:44.127913+0000","last_active":"2024-09-13T15:05:15.108700+0000","last_peered":"2024-09-13T15:05:15.108700+0000","last_clean":"2024-09-13T15:05:15.108700+0000","last_became_active":"2024-09-13T15:03:44.121626+0000","last_became_peered":"2024-09-13T15:03:44.121626+0000","last_unstale":"2024-09-13T15:05:15.108700+0000","last_undegraded":"2024-09-13T15:05:15.108700+0000","last_fullsized":"2024-09-13T15:05:15.108700+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:24.441513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":388,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.287041+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:05:16.287041+0000","last_peered":"2024-09-13T15:05:16.287041+0000","last_clean":"2024-09-13T15:05:16.287041+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:05:16.287041+0000","last_undegraded":"2024-09-13T15:05:16.287041+0000","last_fullsized":"2024-09-13T15:05:16.287041+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107355+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:05:15.107355+0000","last_peered":"2024-09-13T15:05:15.107355+0000","last_clean":"2024-09-13T15:05:15.107355+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:05:15.107355+0000","last_undegraded":"2024-09-13T15:05:15.107355+0000","last_fullsized":"2024-09-13T15:05:15.107355+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107730+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T15:05:15.107730+0000","last_peered":"2024-09-13T15:05:15.107730+0000","last_clean":"2024-09-13T15:05:15.107730+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T15:05:15.107730+0000","last_undegraded":"2024-09-13T15:05:15.107730+0000","last_fullsized":"2024-09-13T15:05:15.107730+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":96,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107768+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:05:15.107768+0000","last_peered":"2024-09-13T15:05:15.107768+0000","last_clean":"2024-09-13T15:05:15.107768+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:05:15.107768+0000","last_undegraded":"2024-09-13T15:05:15.107768+0000","last_fullsized":"2024-09-13T15:05:15.107768+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":544,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109227+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:05:15.109227+0000","last_peered":"2024-09-13T15:05:15.109227+0000","last_clean":"2024-09-13T15:05:15.109227+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:05:15.109227+0000","last_undegraded":"2024-09-13T15:05:15.109227+0000","last_fullsized":"2024-09-13T15:05:15.109227+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":603,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538912+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:05:22.538912+0000","last_peered":"2024-09-13T15:05:22.538912+0000","last_clean":"2024-09-13T15:05:22.538912+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:05:22.538912+0000","last_undegraded":"2024-09-13T15:05:22.538912+0000","last_fullsized":"2024-09-13T15:05:22.538912+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":163,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.109100+0000","last_change":"2024-09-13T15:01:33.853899+0000","last_active":"2024-09-13T15:05:15.109100+0000","last_peered":"2024-09-13T15:05:15.109100+0000","last_clean":"2024-09-13T15:05:15.109100+0000","last_became_active":"2024-09-13T15:01:33.853515+0000","last_became_peered":"2024-09-13T15:01:33.853515+0000","last_unstale":"2024-09-13T15:05:15.109100+0000","last_undegraded":"2024-09-13T15:05:15.109100+0000","last_fullsized":"2024-09-13T15:05:15.109100+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:04.035329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":120,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286364+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:05:16.286364+0000","last_peered":"2024-09-13T15:05:16.286364+0000","last_clean":"2024-09-13T15:05:16.286364+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:05:16.286364+0000","last_undegraded":"2024-09-13T15:05:16.286364+0000","last_fullsized":"2024-09-13T15:05:16.286364+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":253,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636422+0000","last_change":"2024-09-13T15:03:44.113646+0000","last_active":"2024-09-13T15:05:20.636422+0000","last_peered":"2024-09-13T15:05:20.636422+0000","last_clean":"2024-09-13T15:05:20.636422+0000","last_became_active":"2024-09-13T15:03:44.113342+0000","last_became_peered":"2024-09-13T15:03:44.113342+0000","last_unstale":"2024-09-13T15:05:20.636422+0000","last_undegraded":"2024-09-13T15:05:20.636422+0000","last_fullsized":"2024-09-13T15:05:20.636422+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:29.444600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":625,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635921+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:05:20.635921+0000","last_peered":"2024-09-13T15:05:20.635921+0000","last_clean":"2024-09-13T15:05:20.635921+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:05:20.635921+0000","last_undegraded":"2024-09-13T15:05:20.635921+0000","last_fullsized":"2024-09-13T15:05:20.635921+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":375,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:16.286958+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T15:05:16.286958+0000","last_peered":"2024-09-13T15:05:16.286958+0000","last_clean":"2024-09-13T15:05:16.286958+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T15:05:16.286958+0000","last_undegraded":"2024-09-13T15:05:16.286958+0000","last_fullsized":"2024-09-13T15:05:16.286958+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":687,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635325+0000","last_change":"2024-09-13T15:03:44.123370+0000","last_active":"2024-09-13T15:05:20.635325+0000","last_peered":"2024-09-13T15:05:20.635325+0000","last_clean":"2024-09-13T15:05:20.635325+0000","last_became_active":"2024-09-13T15:03:44.122679+0000","last_became_peered":"2024-09-13T15:03:44.122679+0000","last_unstale":"2024-09-13T15:05:20.635325+0000","last_undegraded":"2024-09-13T15:05:20.635325+0000","last_fullsized":"2024-09-13T15:05:20.635325+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:24.806181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":395,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635612+0000","last_change":"2024-09-13T15:03:44.127512+0000","last_active":"2024-09-13T15:05:20.635612+0000","last_peered":"2024-09-13T15:05:20.635612+0000","last_clean":"2024-09-13T15:05:20.635612+0000","last_became_active":"2024-09-13T15:03:44.126735+0000","last_became_peered":"2024-09-13T15:03:44.126735+0000","last_unstale":"2024-09-13T15:05:20.635612+0000","last_undegraded":"2024-09-13T15:05:20.635612+0000","last_fullsized":"2024-09-13T15:05:20.635612+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:20.140202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":636,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.108279+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:05:15.108279+0000","last_peered":"2024-09-13T15:05:15.108279+0000","last_clean":"2024-09-13T15:05:15.108279+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:05:15.108279+0000","last_undegraded":"2024-09-13T15:05:15.108279+0000","last_fullsized":"2024-09-13T15:05:15.108279+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":638,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635905+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:05:20.635905+0000","last_peered":"2024-09-13T15:05:20.635905+0000","last_clean":"2024-09-13T15:05:20.635905+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:05:20.635905+0000","last_undegraded":"2024-09-13T15:05:20.635905+0000","last_fullsized":"2024-09-13T15:05:20.635905+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107411+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T15:05:15.107411+0000","last_peered":"2024-09-13T15:05:15.107411+0000","last_clean":"2024-09-13T15:05:15.107411+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T15:05:15.107411+0000","last_undegraded":"2024-09-13T15:05:15.107411+0000","last_fullsized":"2024-09-13T15:05:15.107411+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:59.608331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":653,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539409+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:05:22.539409+0000","last_peered":"2024-09-13T15:05:22.539409+0000","last_clean":"2024-09-13T15:05:22.539409+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:05:22.539409+0000","last_undegraded":"2024-09-13T15:05:22.539409+0000","last_fullsized":"2024-09-13T15:05:22.539409+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":72,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107602+0000","last_change":"2024-09-13T15:03:44.124264+0000","last_active":"2024-09-13T15:05:15.107602+0000","last_peered":"2024-09-13T15:05:15.107602+0000","last_clean":"2024-09-13T15:05:15.107602+0000","last_became_active":"2024-09-13T15:03:44.123914+0000","last_became_peered":"2024-09-13T15:03:44.123914+0000","last_unstale":"2024-09-13T15:05:15.107602+0000","last_undegraded":"2024-09-13T15:05:15.107602+0000","last_fullsized":"2024-09-13T15:05:15.107602+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:55.477300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107560+0000","last_change":"2024-09-13T15:04:24.852045+0000","last_active":"2024-09-13T15:05:15.107560+0000","last_peered":"2024-09-13T15:05:15.107560+0000","last_clean":"2024-09-13T15:05:15.107560+0000","last_became_active":"2024-09-13T15:04:24.851772+0000","last_became_peered":"2024-09-13T15:04:24.851772+0000","last_unstale":"2024-09-13T15:05:15.107560+0000","last_undegraded":"2024-09-13T15:05:15.107560+0000","last_fullsized":"2024-09-13T15:05:15.107560+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:53:57.699649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":368,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107665+0000","last_change":"2024-09-13T15:01:33.852507+0000","last_active":"2024-09-13T15:05:15.107665+0000","last_peered":"2024-09-13T15:05:15.107665+0000","last_clean":"2024-09-13T15:05:15.107665+0000","last_became_active":"2024-09-13T15:01:33.852196+0000","last_became_peered":"2024-09-13T15:01:33.852196+0000","last_unstale":"2024-09-13T15:05:15.107665+0000","last_undegraded":"2024-09-13T15:05:15.107665+0000","last_fullsized":"2024-09-13T15:05:15.107665+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:28.217977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":106,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107343+0000","last_change":"2024-09-13T15:03:44.116627+0000","last_active":"2024-09-13T15:05:15.107343+0000","last_peered":"2024-09-13T15:05:15.107343+0000","last_clean":"2024-09-13T15:05:15.107343+0000","last_became_active":"2024-09-13T15:03:44.116197+0000","last_became_peered":"2024-09-13T15:03:44.116197+0000","last_unstale":"2024-09-13T15:05:15.107343+0000","last_undegraded":"2024-09-13T15:05:15.107343+0000","last_fullsized":"2024-09-13T15:05:15.107343+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:30.519779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":353,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107950+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:05:15.107950+0000","last_peered":"2024-09-13T15:05:15.107950+0000","last_clean":"2024-09-13T15:05:15.107950+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:05:15.107950+0000","last_undegraded":"2024-09-13T15:05:15.107950+0000","last_fullsized":"2024-09-13T15:05:15.107950+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":689,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.635338+0000","last_change":"2024-09-13T15:03:44.120794+0000","last_active":"2024-09-13T15:05:20.635338+0000","last_peered":"2024-09-13T15:05:20.635338+0000","last_clean":"2024-09-13T15:05:20.635338+0000","last_became_active":"2024-09-13T15:03:44.120431+0000","last_became_peered":"2024-09-13T15:03:44.120431+0000","last_unstale":"2024-09-13T15:05:20.635338+0000","last_undegraded":"2024-09-13T15:05:20.635338+0000","last_fullsized":"2024-09-13T15:05:20.635338+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:27.396753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":648,"reported_epoch":399,"state":"active+clean","last_fresh":"2024-09-13T15:05:20.636327+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:05:20.636327+0000","last_peered":"2024-09-13T15:05:20.636327+0000","last_clean":"2024-09-13T15:05:20.636327+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:05:20.636327+0000","last_undegraded":"2024-09-13T15:05:20.636327+0000","last_fullsized":"2024-09-13T15:05:20.636327+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":86,"reported_epoch":398,"state":"active+clean","last_fresh":"2024-09-13T15:05:15.107274+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T15:05:15.107274+0000","last_peered":"2024-09-13T15:05:15.107274+0000","last_clean":"2024-09-13T15:05:15.107274+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T15:05:15.107274+0000","last_undegraded":"2024-09-13T15:05:15.107274+0000","last_fullsized":"2024-09-13T15:05:15.107274+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":85,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539576+0000","last_change":"2024-09-13T15:03:44.120052+0000","last_active":"2024-09-13T15:05:22.539576+0000","last_peered":"2024-09-13T15:05:22.539576+0000","last_clean":"2024-09-13T15:05:22.539576+0000","last_became_active":"2024-09-13T15:03:44.109611+0000","last_became_peered":"2024-09-13T15:03:44.109611+0000","last_unstale":"2024-09-13T15:05:22.539576+0000","last_undegraded":"2024-09-13T15:05:22.539576+0000","last_fullsized":"2024-09-13T15:05:22.539576+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:04:55.196794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234543,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1330152,"kb_used_data":5468,"kb_used_omap":1,"kb_used_meta":1324670,"kb_avail":92415000,"statfs":{"total":95995035648,"available":94632960000,"internally_reserved":0,"allocated":5599232,"data_stored":3440203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1356462539},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496181,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1241972,"kb_used_data":5160,"kb_used_omap":1,"kb_used_meta":1236798,"kb_avail":92503180,"statfs":{"total":95995035648,"available":94723256320,"internally_reserved":0,"allocated":5283840,"data_stored":3121431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1266481607},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757819,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1384732,"kb_used_data":5712,"kb_used_omap":1,"kb_used_meta":1379006,"kb_avail":92360420,"statfs":{"total":95995035648,"available":94577070080,"internally_reserved":0,"allocated":5849088,"data_stored":3698349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1412102598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019455,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1167540,"kb_used_data":5160,"kb_used_omap":1,"kb_used_meta":1162366,"kb_avail":92577612,"statfs":{"total":95995035648,"available":94799474688,"internally_reserved":0,"allocated":5283840,"data_stored":3120906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1190263238},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650667,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":929828,"kb_used_data":5720,"kb_used_omap":4,"kb_used_meta":924091,"kb_avail":92815324,"statfs":{"total":95995035648,"available":95042891776,"internally_reserved":0,"allocated":5857280,"data_stored":3706474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":946269743},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879602,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":937260,"kb_used_data":5408,"kb_used_omap":6,"kb_used_meta":931833,"kb_avail":92807892,"statfs":{"total":95995035648,"available":95035281408,"internally_reserved":0,"allocated":5537792,"data_stored":3379410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":954197561},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435618,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1111824,"kb_used_data":5124,"kb_used_omap":5,"kb_used_meta":1106682,"kb_avail":92633328,"statfs":{"total":95995035648,"available":94856527872,"internally_reserved":0,"allocated":5246976,"data_stored":3107148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1133242721},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":0,"seq":1494648619062,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:05:29.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:05:29.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:29.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:29.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:29.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:29.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:29.460 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:29.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 5 16 15486 15470 193.359 181.188 0.00327242 0.00516514 2024-09-13T15:05:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:29.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:29.601 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:29.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:29.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:29.736 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:29.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:29.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:29.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:29 smithi042 ceph-mon[108570]: pgmap v1432: 99 pgs: 99 active+clean; 479 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2024-09-13T15:05:29.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:29 smithi042 ceph-mon[113926]: pgmap v1432: 99 pgs: 99 active+clean; 479 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2024-09-13T15:05:29.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:29] "GET /metrics HTTP/1.1" 200 36461 "" "Prometheus/2.51.0" 2024-09-13T15:05:29.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:29.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:29.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:29.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:29 smithi159 ceph-mon[102519]: pgmap v1432: 99 pgs: 99 active+clean; 479 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 56 MiB/s wr, 891 op/s 2024-09-13T15:05:30.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:30.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:30.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:30.251 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.280 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:30.381 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:30.406 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:30.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 6 16 17949 17933 186.787 153.938 0.00847318 0.00534817 2024-09-13T15:05:30.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:30.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:30.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:30.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:30.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:30.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:30.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:30.943 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:30.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:31.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.083 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:31.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:31.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:31.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:31.317 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:31.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:31.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:31.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 7 16 20079 20063 179.119 133.125 0.00560778 0.00557588 2024-09-13T15:05:31.488 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:31.590 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:31.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:31.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:31.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:31.767 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:31.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:31.901 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:31.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:32.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:32.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:32.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:31 smithi042 ceph-mon[113926]: pgmap v1433: 99 pgs: 99 active+clean; 596 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T15:05:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:31 smithi042 ceph-mon[108570]: pgmap v1433: 99 pgs: 99 active+clean; 596 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T15:05:32.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:32.174 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:31 smithi159 ceph-mon[102519]: pgmap v1433: 99 pgs: 99 active+clean; 596 MiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 60 MiB/s wr, 952 op/s 2024-09-13T15:05:32.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:32.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.312 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:32.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:32.414 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:32.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:32.461 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 8 16 22262 22246 173.782 136.438 0.00710464 0.00574722 2024-09-13T15:05:32.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:32.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:32.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:32.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:32.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:32.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:32.860 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:32.962 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:32.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:32.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:33.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:33.134 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:33.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:33.236 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:33.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:33.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:33.378 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:33.400 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:33.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 9 16 24373 24357 169.125 131.938 0.00534291 0.00590683 2024-09-13T15:05:33.501 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:33.535 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:33.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:33.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:33.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:33.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:33.795 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:33.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:33.930 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:33.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:34.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.071 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:34.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:33 smithi042 ceph-mon[113926]: pgmap v1434: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T15:05:34.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:33 smithi042 ceph-mon[108570]: pgmap v1434: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T15:05:34.173 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:34.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:33 smithi159 ceph-mon[102519]: pgmap v1434: 99 pgs: 99 active+clean; 1.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s 2024-09-13T15:05:34.305 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:34.350 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:34.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:34.452 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:34.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 10 16 26529 26513 165.686 134.75 0.00522708 0.00603011 2024-09-13T15:05:34.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:34.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:34.623 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:34.726 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:34.763 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:34.865 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:34.902 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:34.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:35.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:35.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:35.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:35.132 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:05:35.132 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:05:35.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:35.171 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:35.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:35.273 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:35.312 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:35.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:35.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:35.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:35.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 11 16 28670 28654 162.787 133.812 0.00192995 0.00612174 2024-09-13T15:05:35.548 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:35.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:35.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:35.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:35.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:35.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.848 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:35.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:35.950 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:35.986 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:35.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:35.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:36.088 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:35 smithi042 ceph-mon[108570]: pgmap v1435: 99 pgs: 99 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T15:05:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:35 smithi042 ceph-mon[113926]: pgmap v1435: 99 pgs: 99 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T15:05:36.130 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:36.233 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:35 smithi159 ceph-mon[102519]: pgmap v1435: 99 pgs: 99 active+clean; 1.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-09-13T15:05:36.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:36.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:36.409 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:36.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 12 16 30817 30801 160.403 134.188 0.0139144 0.00622879 2024-09-13T15:05:36.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:36.547 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:36.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:36.649 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:36.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:36.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:36.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:36.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:36.921 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:36.921 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:36.921 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:36.922 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:36.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:36.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:36.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:36.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:37.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.099 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:37.235 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:05:37.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:37.337 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:37.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:37.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:37.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 13 16 33030 33014 158.703 138.312 0.00393783 0.00629577 2024-09-13T15:05:37.485 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:37.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:37.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:37.659 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:37.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:37.788 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:37.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:37.890 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:37.927 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:37.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:37.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.057 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:37 smithi042 ceph-mon[113926]: pgmap v1436: 99 pgs: 99 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T15:05:38.057 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:37 smithi042 ceph-mon[108570]: pgmap v1436: 99 pgs: 99 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T15:05:38.061 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:38.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:38.163 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:38.204 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:38.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:37 smithi159 ceph-mon[102519]: pgmap v1436: 99 pgs: 99 active+clean; 1.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-13T15:05:38.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:38.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:38.430 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:05:38.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:38.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 14 16 35131 35115 156.744 131.312 0.00319084 0.00637458 2024-09-13T15:05:38.466 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:38.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:38.480 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:38.568 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:38.597 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:38.695 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:05:38.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:38.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:38.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:38.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:38.894 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:38.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:38.996 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:39.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.026 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:39.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:39.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:39.174 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:39.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.332 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:39.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:39.434 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:39.458 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:39.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 15 16 36741 36725 153.003 100.625 0.0130196 0.00652906 2024-09-13T15:05:39.559 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.597 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:39.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:39.698 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:39.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:39.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:39.819 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:05:39.830 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:39.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:39.951 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[113926]: pgmap v1437: 99 pgs: 99 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-13T15:05:39.951 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/987005219' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:39.952 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:39.952 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[108570]: pgmap v1437: 99 pgs: 99 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-13T15:05:39.952 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/987005219' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:39.952 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:39.952 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:39] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-09-13T15:05:39.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:39.987 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:39.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:39.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:40.089 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:40.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:40.232 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:39 smithi159 ceph-mon[102519]: pgmap v1437: 99 pgs: 99 active+clean; 1.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-09-13T15:05:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:39 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/987005219' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:39 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:05:40.254 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:40.356 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:40.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.399 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:40.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:40.464 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 16 16 38454 38438 150.131 107.062 0.0113939 0.00665551 2024-09-13T15:05:40.500 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:40.514 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:40.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:40.539 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:40.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.676 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.778 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:40.812 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:40.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.813 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:40.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:40.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:40.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:40.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:41.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:41.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:41.023 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[108570]: osdmap e401: 8 total, 8 up, 7 in 2024-09-13T15:05:41.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:41.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:41.024 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:40 smithi042 ceph-mon[113926]: osdmap e401: 8 total, 8 up, 7 in 2024-09-13T15:05:41.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:41.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:41.190 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:41.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:40 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:05:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:40 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:05:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:40 smithi159 ceph-mon[102519]: osdmap e401: 8 total, 8 up, 7 in 2024-09-13T15:05:41.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.380 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:41.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 17 16 40357 40341 148.295 118.938 0.0146686 0.00673789 2024-09-13T15:05:41.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.522 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:41.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.624 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:41.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:41.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:41.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:41.795 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:41.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:41.896 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:41.933 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:41.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:42.035 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:42.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.073 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:42.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:42.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:41 smithi042 ceph-mon[113926]: pgmap v1439: 99 pgs: 99 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T15:05:42.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:41 smithi042 ceph-mon[108570]: pgmap v1439: 99 pgs: 99 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T15:05:42.174 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:42.205 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:41 smithi159 ceph-mon[102519]: pgmap v1439: 99 pgs: 99 active+clean; 2.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 145 MiB/s wr, 2.32k op/s 2024-09-13T15:05:42.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.336 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:42.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:42.462 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 18 16 42354 42338 146.99 124.812 0.00939785 0.00679803 2024-09-13T15:05:42.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.471 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:42.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:42.572 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:42.601 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:42.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:42.703 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:42.740 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:42.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:42.843 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:42.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:42.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:42.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:42.983 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:43.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:43.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:43.025 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:05:43.120 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.167 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:43.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:43.268 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:43.298 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:43.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:43.400 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:43.429 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:43.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 19 16 44205 44189 145.342 115.688 0.00816282 0.00687485 2024-09-13T15:05:43.532 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:43.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.576 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:43.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:43.677 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:43.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:43.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:43.811 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:43.853 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:43.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:43.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:43.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:43.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:43.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:44.098 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:44.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:44.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:44.165 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:05:44.165 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:43 smithi042 ceph-mon[108570]: pgmap v1440: 99 pgs: 99 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2024-09-13T15:05:44.166 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:43 smithi042 ceph-mon[113926]: pgmap v1440: 99 pgs: 99 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2024-09-13T15:05:44.168 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:05:44.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:43 smithi159 ceph-mon[102519]: pgmap v1440: 99 pgs: 99 active+clean; 2.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 129 MiB/s wr, 2.07k op/s 2024-09-13T15:05:44.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:44.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:44.389 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:44.423 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:44.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:44.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:05:44.462566+0000 min lat: 0.00110032 max lat: 0.182233 avg lat: 0.0069601 2024-09-13T15:05:44.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:05:44.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 20 16 45965 45949 143.575 110 0.00493484 0.0069601 2024-09-13T15:05:44.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:44.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:44.666 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:44.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:44.817 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:44.851 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:44.953 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:44.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:44.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:44.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:45.088 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:44 smithi042 ceph-mon[113926]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:45.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:44 smithi042 ceph-mon[108570]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:45.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:45.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:45.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:45.234 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:44 smithi159 ceph-mon[102519]: from='client.36153 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:05:45.263 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:45.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:45.393 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1440,"stamp":"2024-09-13T15:05:42.717920+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2568296421,"num_objects":39206,"num_object_clones":0,"num_object_copies":117618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39244,"num_write_kb":2507553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40878,"ondisk_log_size":40878,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15517616,"kb_used_data":7295596,"kb_used_omap":22,"kb_used_meta":8221929,"kb_avail":640698448,"statfs":{"total":671965249536,"available":656075210752,"internally_reserved":0,"allocated":7470690304,"data_stored":7455549838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":8419255527},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1357316096,"num_objects":20711,"num_object_clones":0,"num_object_copies":62133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20711,"num_write_kb":1325504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003978"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":629,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.836700+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:05:39.836700+0000","last_peered":"2024-09-13T15:05:39.836700+0000","last_clean":"2024-09-13T15:05:39.836700+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:05:39.836700+0000","last_undegraded":"2024-09-13T15:05:39.836700+0000","last_fullsized":"2024-09-13T15:05:39.836700+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":94,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717151+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:05:22.717151+0000","last_peered":"2024-09-13T15:05:22.717151+0000","last_clean":"2024-09-13T15:05:22.717151+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:05:22.717151+0000","last_undegraded":"2024-09-13T15:05:22.717151+0000","last_fullsized":"2024-09-13T15:05:22.717151+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":193,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834201+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:05:39.834201+0000","last_peered":"2024-09-13T15:05:39.834201+0000","last_clean":"2024-09-13T15:05:39.834201+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:05:39.834201+0000","last_undegraded":"2024-09-13T15:05:39.834201+0000","last_fullsized":"2024-09-13T15:05:39.834201+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":398,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834965+0000","last_change":"2024-09-13T15:03:44.123745+0000","last_active":"2024-09-13T15:05:39.834965+0000","last_peered":"2024-09-13T15:05:39.834965+0000","last_clean":"2024-09-13T15:05:39.834965+0000","last_became_active":"2024-09-13T15:03:44.123397+0000","last_became_peered":"2024-09-13T15:03:44.123397+0000","last_unstale":"2024-09-13T15:05:39.834965+0000","last_undegraded":"2024-09-13T15:05:39.834965+0000","last_fullsized":"2024-09-13T15:05:39.834965+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.079957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833878+0000","last_change":"2024-09-13T15:03:44.128445+0000","last_active":"2024-09-13T15:05:39.833878+0000","last_peered":"2024-09-13T15:05:39.833878+0000","last_clean":"2024-09-13T15:05:39.833878+0000","last_became_active":"2024-09-13T15:03:44.128047+0000","last_became_peered":"2024-09-13T15:03:44.128047+0000","last_unstale":"2024-09-13T15:05:39.833878+0000","last_undegraded":"2024-09-13T15:05:39.833878+0000","last_fullsized":"2024-09-13T15:05:39.833878+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:22.359631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716665+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:05:22.716665+0000","last_peered":"2024-09-13T15:05:22.716665+0000","last_clean":"2024-09-13T15:05:22.716665+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:05:22.716665+0000","last_undegraded":"2024-09-13T15:05:22.716665+0000","last_fullsized":"2024-09-13T15:05:22.716665+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":655,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834082+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:05:39.834082+0000","last_peered":"2024-09-13T15:05:39.834082+0000","last_clean":"2024-09-13T15:05:39.834082+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:05:39.834082+0000","last_undegraded":"2024-09-13T15:05:39.834082+0000","last_fullsized":"2024-09-13T15:05:39.834082+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":494,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424562+0000","last_change":"2024-09-13T15:03:44.130129+0000","last_active":"2024-09-13T15:05:23.424562+0000","last_peered":"2024-09-13T15:05:23.424562+0000","last_clean":"2024-09-13T15:05:23.424562+0000","last_became_active":"2024-09-13T15:03:44.114883+0000","last_became_peered":"2024-09-13T15:03:44.114883+0000","last_unstale":"2024-09-13T15:05:23.424562+0000","last_undegraded":"2024-09-13T15:05:23.424562+0000","last_fullsized":"2024-09-13T15:05:23.424562+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:54.544220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"401'2555","reported_seq":2568,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.442130+0000","last_change":"2024-09-13T15:05:11.104668+0000","last_active":"2024-09-13T15:05:41.442130+0000","last_peered":"2024-09-13T15:05:41.442130+0000","last_clean":"2024-09-13T15:05:41.442130+0000","last_became_active":"2024-09-13T15:05:11.104349+0000","last_became_peered":"2024-09-13T15:05:11.104349+0000","last_unstale":"2024-09-13T15:05:41.442130+0000","last_undegraded":"2024-09-13T15:05:41.442130+0000","last_fullsized":"2024-09-13T15:05:41.442130+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2555,"log_dups_size":0,"ondisk_log_size":2555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:34.717367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167444480,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2555,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":116,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717154+0000","last_change":"2024-09-13T15:03:44.129600+0000","last_active":"2024-09-13T15:05:22.717154+0000","last_peered":"2024-09-13T15:05:22.717154+0000","last_clean":"2024-09-13T15:05:22.717154+0000","last_became_active":"2024-09-13T15:03:44.123858+0000","last_became_peered":"2024-09-13T15:03:44.123858+0000","last_unstale":"2024-09-13T15:05:22.717154+0000","last_undegraded":"2024-09-13T15:05:22.717154+0000","last_fullsized":"2024-09-13T15:05:22.717154+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:15.345841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":59,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424434+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:05:23.424434+0000","last_peered":"2024-09-13T15:05:23.424434+0000","last_clean":"2024-09-13T15:05:23.424434+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:05:23.424434+0000","last_undegraded":"2024-09-13T15:05:23.424434+0000","last_fullsized":"2024-09-13T15:05:23.424434+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"401'2648","reported_seq":2671,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:42.275863+0000","last_change":"2024-09-13T15:05:11.103932+0000","last_active":"2024-09-13T15:05:42.275863+0000","last_peered":"2024-09-13T15:05:42.275863+0000","last_clean":"2024-09-13T15:05:42.275863+0000","last_became_active":"2024-09-13T15:05:11.103484+0000","last_became_peered":"2024-09-13T15:05:11.103484+0000","last_unstale":"2024-09-13T15:05:42.275863+0000","last_undegraded":"2024-09-13T15:05:42.275863+0000","last_fullsized":"2024-09-13T15:05:42.275863+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2648,"log_dups_size":0,"ondisk_log_size":2648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:53.861057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173539328,"num_objects":2648,"num_object_clones":0,"num_object_copies":7944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2648,"num_write_kb":169472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":92,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717140+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T15:05:22.717140+0000","last_peered":"2024-09-13T15:05:22.717140+0000","last_clean":"2024-09-13T15:05:22.717140+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T15:05:22.717140+0000","last_undegraded":"2024-09-13T15:05:22.717140+0000","last_fullsized":"2024-09-13T15:05:22.717140+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:20.897906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716981+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T15:05:22.716981+0000","last_peered":"2024-09-13T15:05:22.716981+0000","last_clean":"2024-09-13T15:05:22.716981+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T15:05:22.716981+0000","last_undegraded":"2024-09-13T15:05:22.716981+0000","last_fullsized":"2024-09-13T15:05:22.716981+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"400'2032","reported_seq":2053,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:37.761443+0000","last_change":"2024-09-13T15:05:11.537086+0000","last_active":"2024-09-13T15:05:37.761443+0000","last_peered":"2024-09-13T15:05:37.761443+0000","last_clean":"2024-09-13T15:05:37.761443+0000","last_became_active":"2024-09-13T15:05:11.536730+0000","last_became_peered":"2024-09-13T15:05:11.536730+0000","last_unstale":"2024-09-13T15:05:37.761443+0000","last_undegraded":"2024-09-13T15:05:37.761443+0000","last_fullsized":"2024-09-13T15:05:37.761443+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2032,"log_dups_size":0,"ondisk_log_size":2032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:51.566063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133169152,"num_objects":2032,"num_object_clones":0,"num_object_copies":6096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2032,"num_write_kb":130048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":71,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424766+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:05:23.424766+0000","last_peered":"2024-09-13T15:05:23.424766+0000","last_clean":"2024-09-13T15:05:23.424766+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:05:23.424766+0000","last_undegraded":"2024-09-13T15:05:23.424766+0000","last_fullsized":"2024-09-13T15:05:23.424766+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":376,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424591+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T15:05:23.424591+0000","last_peered":"2024-09-13T15:05:23.424591+0000","last_clean":"2024-09-13T15:05:23.424591+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T15:05:23.424591+0000","last_undegraded":"2024-09-13T15:05:23.424591+0000","last_fullsized":"2024-09-13T15:05:23.424591+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"401'2734","reported_seq":2757,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:42.276630+0000","last_change":"2024-09-13T15:05:11.101187+0000","last_active":"2024-09-13T15:05:42.276630+0000","last_peered":"2024-09-13T15:05:42.276630+0000","last_clean":"2024-09-13T15:05:42.276630+0000","last_became_active":"2024-09-13T15:05:11.100618+0000","last_became_peered":"2024-09-13T15:05:11.100618+0000","last_unstale":"2024-09-13T15:05:42.276630+0000","last_undegraded":"2024-09-13T15:05:42.276630+0000","last_fullsized":"2024-09-13T15:05:42.276630+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2734,"log_dups_size":0,"ondisk_log_size":2734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:13.595199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2734,"num_write_kb":174976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":194,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.837104+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:05:39.837104+0000","last_peered":"2024-09-13T15:05:39.837104+0000","last_clean":"2024-09-13T15:05:39.837104+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:05:39.837104+0000","last_undegraded":"2024-09-13T15:05:39.837104+0000","last_fullsized":"2024-09-13T15:05:39.837104+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":432,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817590+0000","last_change":"2024-09-13T15:03:44.122648+0000","last_active":"2024-09-13T15:05:39.817590+0000","last_peered":"2024-09-13T15:05:39.817590+0000","last_clean":"2024-09-13T15:05:39.817590+0000","last_became_active":"2024-09-13T15:03:44.121071+0000","last_became_peered":"2024-09-13T15:03:44.121071+0000","last_unstale":"2024-09-13T15:05:39.817590+0000","last_undegraded":"2024-09-13T15:05:39.817590+0000","last_fullsized":"2024-09-13T15:05:39.817590+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:51.385950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"400'2106","reported_seq":2127,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:37.796113+0000","last_change":"2024-09-13T15:05:11.103812+0000","last_active":"2024-09-13T15:05:37.796113+0000","last_peered":"2024-09-13T15:05:37.796113+0000","last_clean":"2024-09-13T15:05:37.796113+0000","last_became_active":"2024-09-13T15:05:11.103500+0000","last_became_peered":"2024-09-13T15:05:11.103500+0000","last_unstale":"2024-09-13T15:05:37.796113+0000","last_undegraded":"2024-09-13T15:05:37.796113+0000","last_fullsized":"2024-09-13T15:05:37.796113+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2106,"log_dups_size":0,"ondisk_log_size":2106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:33.535670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138018816,"num_objects":2106,"num_object_clones":0,"num_object_copies":6318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2106,"num_write_kb":134784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":981,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.835321+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:05:39.835321+0000","last_peered":"2024-09-13T15:05:39.835321+0000","last_clean":"2024-09-13T15:05:39.835321+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:05:39.835321+0000","last_undegraded":"2024-09-13T15:05:39.835321+0000","last_fullsized":"2024-09-13T15:05:39.835321+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":704,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.836566+0000","last_change":"2024-09-13T15:03:44.133692+0000","last_active":"2024-09-13T15:05:39.836566+0000","last_peered":"2024-09-13T15:05:39.836566+0000","last_clean":"2024-09-13T15:05:39.836566+0000","last_became_active":"2024-09-13T15:03:44.125117+0000","last_became_peered":"2024-09-13T15:03:44.125117+0000","last_unstale":"2024-09-13T15:05:39.836566+0000","last_undegraded":"2024-09-13T15:05:39.836566+0000","last_fullsized":"2024-09-13T15:05:39.836566+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:12.857175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717035+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:05:22.717035+0000","last_peered":"2024-09-13T15:05:22.717035+0000","last_clean":"2024-09-13T15:05:22.717035+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:05:22.717035+0000","last_undegraded":"2024-09-13T15:05:22.717035+0000","last_fullsized":"2024-09-13T15:05:22.717035+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"401'2513","reported_seq":2527,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.489477+0000","last_change":"2024-09-13T15:05:11.537076+0000","last_active":"2024-09-13T15:05:41.489477+0000","last_peered":"2024-09-13T15:05:41.489477+0000","last_clean":"2024-09-13T15:05:41.489477+0000","last_became_active":"2024-09-13T15:05:11.536775+0000","last_became_peered":"2024-09-13T15:05:11.536775+0000","last_unstale":"2024-09-13T15:05:41.489477+0000","last_undegraded":"2024-09-13T15:05:41.489477+0000","last_fullsized":"2024-09-13T15:05:41.489477+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2513,"log_dups_size":0,"ondisk_log_size":2513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:38.213878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164691968,"num_objects":2513,"num_object_clones":0,"num_object_copies":7539,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2513,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2513,"num_write_kb":160832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":476,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538781+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:05:22.538781+0000","last_peered":"2024-09-13T15:05:22.538781+0000","last_clean":"2024-09-13T15:05:22.538781+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:05:22.538781+0000","last_undegraded":"2024-09-13T15:05:22.538781+0000","last_fullsized":"2024-09-13T15:05:22.538781+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":424,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424737+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T15:05:23.424737+0000","last_peered":"2024-09-13T15:05:23.424737+0000","last_clean":"2024-09-13T15:05:23.424737+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T15:05:23.424737+0000","last_undegraded":"2024-09-13T15:05:23.424737+0000","last_fullsized":"2024-09-13T15:05:23.424737+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717064+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T15:05:22.717064+0000","last_peered":"2024-09-13T15:05:22.717064+0000","last_clean":"2024-09-13T15:05:22.717064+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T15:05:22.717064+0000","last_undegraded":"2024-09-13T15:05:22.717064+0000","last_fullsized":"2024-09-13T15:05:22.717064+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"401'2446","reported_seq":2461,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.442690+0000","last_change":"2024-09-13T15:05:11.105322+0000","last_active":"2024-09-13T15:05:41.442690+0000","last_peered":"2024-09-13T15:05:41.442690+0000","last_clean":"2024-09-13T15:05:41.442690+0000","last_became_active":"2024-09-13T15:05:11.104691+0000","last_became_peered":"2024-09-13T15:05:11.104691+0000","last_unstale":"2024-09-13T15:05:41.442690+0000","last_undegraded":"2024-09-13T15:05:41.442690+0000","last_fullsized":"2024-09-13T15:05:41.442690+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2446,"log_dups_size":0,"ondisk_log_size":2446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:23.376418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160301056,"num_objects":2446,"num_object_clones":0,"num_object_copies":7338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2446,"num_write_kb":156544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716695+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:05:22.716695+0000","last_peered":"2024-09-13T15:05:22.716695+0000","last_clean":"2024-09-13T15:05:22.716695+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:05:22.716695+0000","last_undegraded":"2024-09-13T15:05:22.716695+0000","last_fullsized":"2024-09-13T15:05:22.716695+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717172+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T15:05:22.717172+0000","last_peered":"2024-09-13T15:05:22.717172+0000","last_clean":"2024-09-13T15:05:22.717172+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T15:05:22.717172+0000","last_undegraded":"2024-09-13T15:05:22.717172+0000","last_fullsized":"2024-09-13T15:05:22.717172+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"401'2552","reported_seq":2567,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.468617+0000","last_change":"2024-09-13T15:05:11.103992+0000","last_active":"2024-09-13T15:05:41.468617+0000","last_peered":"2024-09-13T15:05:41.468617+0000","last_clean":"2024-09-13T15:05:41.468617+0000","last_became_active":"2024-09-13T15:05:11.103694+0000","last_became_peered":"2024-09-13T15:05:11.103694+0000","last_unstale":"2024-09-13T15:05:41.468617+0000","last_undegraded":"2024-09-13T15:05:41.468617+0000","last_fullsized":"2024-09-13T15:05:41.468617+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2552,"log_dups_size":0,"ondisk_log_size":2552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:15.405086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167247872,"num_objects":2552,"num_object_clones":0,"num_object_copies":7656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2552,"num_write_kb":163328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":476,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539176+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:05:22.539176+0000","last_peered":"2024-09-13T15:05:22.539176+0000","last_clean":"2024-09-13T15:05:22.539176+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:05:22.539176+0000","last_undegraded":"2024-09-13T15:05:22.539176+0000","last_fullsized":"2024-09-13T15:05:22.539176+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":632,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833498+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:05:39.833498+0000","last_peered":"2024-09-13T15:05:39.833498+0000","last_clean":"2024-09-13T15:05:39.833498+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:05:39.833498+0000","last_undegraded":"2024-09-13T15:05:39.833498+0000","last_fullsized":"2024-09-13T15:05:39.833498+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":645,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.818155+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:05:39.818155+0000","last_peered":"2024-09-13T15:05:39.818155+0000","last_clean":"2024-09-13T15:05:39.818155+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:05:39.818155+0000","last_undegraded":"2024-09-13T15:05:39.818155+0000","last_fullsized":"2024-09-13T15:05:39.818155+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"401'2672","reported_seq":2695,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:42.262717+0000","last_change":"2024-09-13T15:05:11.105485+0000","last_active":"2024-09-13T15:05:42.262717+0000","last_peered":"2024-09-13T15:05:42.262717+0000","last_clean":"2024-09-13T15:05:42.262717+0000","last_became_active":"2024-09-13T15:05:11.105170+0000","last_became_peered":"2024-09-13T15:05:11.105170+0000","last_unstale":"2024-09-13T15:05:42.262717+0000","last_undegraded":"2024-09-13T15:05:42.262717+0000","last_fullsized":"2024-09-13T15:05:42.262717+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2672,"log_dups_size":0,"ondisk_log_size":2672,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:35.316067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175112192,"num_objects":2672,"num_object_clones":0,"num_object_copies":8016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2672,"num_write_kb":171008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":600,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424226+0000","last_change":"2024-09-13T15:03:46.574157+0000","last_active":"2024-09-13T15:05:23.424226+0000","last_peered":"2024-09-13T15:05:23.424226+0000","last_clean":"2024-09-13T15:05:23.424226+0000","last_became_active":"2024-09-13T15:03:46.572750+0000","last_became_peered":"2024-09-13T15:03:46.572750+0000","last_unstale":"2024-09-13T15:05:23.424226+0000","last_undegraded":"2024-09-13T15:05:23.424226+0000","last_fullsized":"2024-09-13T15:05:23.424226+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:55.892848+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":114,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538672+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:05:22.538672+0000","last_peered":"2024-09-13T15:05:22.538672+0000","last_clean":"2024-09-13T15:05:22.538672+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:05:22.538672+0000","last_undegraded":"2024-09-13T15:05:22.538672+0000","last_fullsized":"2024-09-13T15:05:22.538672+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":618,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834808+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:05:39.834808+0000","last_peered":"2024-09-13T15:05:39.834808+0000","last_clean":"2024-09-13T15:05:39.834808+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:05:39.834808+0000","last_undegraded":"2024-09-13T15:05:39.834808+0000","last_fullsized":"2024-09-13T15:05:39.834808+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":386,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424893+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:05:23.424893+0000","last_peered":"2024-09-13T15:05:23.424893+0000","last_clean":"2024-09-13T15:05:23.424893+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:05:23.424893+0000","last_undegraded":"2024-09-13T15:05:23.424893+0000","last_fullsized":"2024-09-13T15:05:23.424893+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"401'2553","reported_seq":2568,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.474354+0000","last_change":"2024-09-13T15:05:11.104165+0000","last_active":"2024-09-13T15:05:41.474354+0000","last_peered":"2024-09-13T15:05:41.474354+0000","last_clean":"2024-09-13T15:05:41.474354+0000","last_became_active":"2024-09-13T15:05:11.103885+0000","last_became_peered":"2024-09-13T15:05:11.103885+0000","last_unstale":"2024-09-13T15:05:41.474354+0000","last_undegraded":"2024-09-13T15:05:41.474354+0000","last_fullsized":"2024-09-13T15:05:41.474354+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2553,"log_dups_size":0,"ondisk_log_size":2553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:32.640246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167313408,"num_objects":2553,"num_object_clones":0,"num_object_copies":7659,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2553,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2553,"num_write_kb":163392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":912,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817485+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:05:39.817485+0000","last_peered":"2024-09-13T15:05:39.817485+0000","last_clean":"2024-09-13T15:05:39.817485+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:05:39.817485+0000","last_undegraded":"2024-09-13T15:05:39.817485+0000","last_fullsized":"2024-09-13T15:05:39.817485+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":637,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817679+0000","last_change":"2024-09-13T15:03:44.117533+0000","last_active":"2024-09-13T15:05:39.817679+0000","last_peered":"2024-09-13T15:05:39.817679+0000","last_clean":"2024-09-13T15:05:39.817679+0000","last_became_active":"2024-09-13T15:03:44.117216+0000","last_became_peered":"2024-09-13T15:03:44.117216+0000","last_unstale":"2024-09-13T15:05:39.817679+0000","last_undegraded":"2024-09-13T15:05:39.817679+0000","last_fullsized":"2024-09-13T15:05:39.817679+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:17.222690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":192,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716575+0000","last_change":"2024-09-13T15:03:44.115491+0000","last_active":"2024-09-13T15:05:22.716575+0000","last_peered":"2024-09-13T15:05:22.716575+0000","last_clean":"2024-09-13T15:05:22.716575+0000","last_became_active":"2024-09-13T15:03:44.114978+0000","last_became_peered":"2024-09-13T15:03:44.114978+0000","last_unstale":"2024-09-13T15:05:22.716575+0000","last_undegraded":"2024-09-13T15:05:22.716575+0000","last_fullsized":"2024-09-13T15:05:22.716575+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:04.996891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":470,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.819112+0000","last_change":"2024-09-13T15:03:44.117741+0000","last_active":"2024-09-13T15:05:39.819112+0000","last_peered":"2024-09-13T15:05:39.819112+0000","last_clean":"2024-09-13T15:05:39.819112+0000","last_became_active":"2024-09-13T15:03:44.117404+0000","last_became_peered":"2024-09-13T15:03:44.117404+0000","last_unstale":"2024-09-13T15:05:39.819112+0000","last_undegraded":"2024-09-13T15:05:39.819112+0000","last_fullsized":"2024-09-13T15:05:39.819112+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:02.169360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"401'2542","reported_seq":2557,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.427523+0000","last_change":"2024-09-13T15:05:11.537213+0000","last_active":"2024-09-13T15:05:41.427523+0000","last_peered":"2024-09-13T15:05:41.427523+0000","last_clean":"2024-09-13T15:05:41.427523+0000","last_became_active":"2024-09-13T15:05:11.536909+0000","last_became_peered":"2024-09-13T15:05:11.536909+0000","last_unstale":"2024-09-13T15:05:41.427523+0000","last_undegraded":"2024-09-13T15:05:41.427523+0000","last_fullsized":"2024-09-13T15:05:41.427523+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2542,"log_dups_size":0,"ondisk_log_size":2542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:13.844350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166592512,"num_objects":2542,"num_object_clones":0,"num_object_copies":7626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2542,"num_write_kb":162688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":648,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716444+0000","last_change":"2024-09-13T15:03:46.571609+0000","last_active":"2024-09-13T15:05:22.716444+0000","last_peered":"2024-09-13T15:05:22.716444+0000","last_clean":"2024-09-13T15:05:22.716444+0000","last_became_active":"2024-09-13T15:03:46.571111+0000","last_became_peered":"2024-09-13T15:03:46.571111+0000","last_unstale":"2024-09-13T15:05:22.716444+0000","last_undegraded":"2024-09-13T15:05:22.716444+0000","last_fullsized":"2024-09-13T15:05:22.716444+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:26.923160+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":686,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538587+0000","last_change":"2024-09-13T15:03:44.113114+0000","last_active":"2024-09-13T15:05:22.538587+0000","last_peered":"2024-09-13T15:05:22.538587+0000","last_clean":"2024-09-13T15:05:22.538587+0000","last_became_active":"2024-09-13T15:03:44.112798+0000","last_became_peered":"2024-09-13T15:03:44.112798+0000","last_unstale":"2024-09-13T15:05:22.538587+0000","last_undegraded":"2024-09-13T15:05:22.538587+0000","last_fullsized":"2024-09-13T15:05:22.538587+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:39.690606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":630,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817909+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:05:39.817909+0000","last_peered":"2024-09-13T15:05:39.817909+0000","last_clean":"2024-09-13T15:05:39.817909+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:05:39.817909+0000","last_undegraded":"2024-09-13T15:05:39.817909+0000","last_fullsized":"2024-09-13T15:05:39.817909+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":698,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.818248+0000","last_change":"2024-09-13T15:03:44.120103+0000","last_active":"2024-09-13T15:05:39.818248+0000","last_peered":"2024-09-13T15:05:39.818248+0000","last_clean":"2024-09-13T15:05:39.818248+0000","last_became_active":"2024-09-13T15:03:44.119715+0000","last_became_peered":"2024-09-13T15:03:44.119715+0000","last_unstale":"2024-09-13T15:05:39.818248+0000","last_undegraded":"2024-09-13T15:05:39.818248+0000","last_fullsized":"2024-09-13T15:05:39.818248+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:16.484960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"401'2393","reported_seq":2408,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:41.484289+0000","last_change":"2024-09-13T15:05:11.104779+0000","last_active":"2024-09-13T15:05:41.484289+0000","last_peered":"2024-09-13T15:05:41.484289+0000","last_clean":"2024-09-13T15:05:41.484289+0000","last_became_active":"2024-09-13T15:05:11.104444+0000","last_became_peered":"2024-09-13T15:05:11.104444+0000","last_unstale":"2024-09-13T15:05:41.484289+0000","last_undegraded":"2024-09-13T15:05:41.484289+0000","last_fullsized":"2024-09-13T15:05:41.484289+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2393,"log_dups_size":0,"ondisk_log_size":2393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:23.452855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2393,"num_write_kb":153152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":439,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538299+0000","last_change":"2024-09-13T15:03:47.227943+0000","last_active":"2024-09-13T15:05:22.538299+0000","last_peered":"2024-09-13T15:05:22.538299+0000","last_clean":"2024-09-13T15:05:22.538299+0000","last_became_active":"2024-09-13T15:03:46.585701+0000","last_became_peered":"2024-09-13T15:03:46.585701+0000","last_unstale":"2024-09-13T15:05:22.538299+0000","last_undegraded":"2024-09-13T15:05:22.538299+0000","last_fullsized":"2024-09-13T15:05:22.538299+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:03:47.226199+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:03:47.226199+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:12.460412+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":121,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.820668+0000","last_change":"2024-09-13T15:01:33.852433+0000","last_active":"2024-09-13T15:05:39.820668+0000","last_peered":"2024-09-13T15:05:39.820668+0000","last_clean":"2024-09-13T15:05:39.820668+0000","last_became_active":"2024-09-13T15:01:33.852015+0000","last_became_peered":"2024-09-13T15:01:33.852015+0000","last_unstale":"2024-09-13T15:05:39.820668+0000","last_undegraded":"2024-09-13T15:05:39.820668+0000","last_fullsized":"2024-09-13T15:05:39.820668+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:52:03.246939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":109,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:31.482573+0000","last_change":"2024-09-13T15:03:44.120307+0000","last_active":"2024-09-13T15:05:31.482573+0000","last_peered":"2024-09-13T15:05:31.482573+0000","last_clean":"2024-09-13T15:05:31.482573+0000","last_became_active":"2024-09-13T15:03:44.119949+0000","last_became_peered":"2024-09-13T15:03:44.119949+0000","last_unstale":"2024-09-13T15:05:31.482573+0000","last_undegraded":"2024-09-13T15:05:31.482573+0000","last_fullsized":"2024-09-13T15:05:31.482573+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.569133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":643,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817496+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:05:39.817496+0000","last_peered":"2024-09-13T15:05:39.817496+0000","last_clean":"2024-09-13T15:05:39.817496+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:05:39.817496+0000","last_undegraded":"2024-09-13T15:05:39.817496+0000","last_fullsized":"2024-09-13T15:05:39.817496+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"401'2341","reported_seq":2364,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:40.008372+0000","last_change":"2024-09-13T15:05:11.105474+0000","last_active":"2024-09-13T15:05:40.008372+0000","last_peered":"2024-09-13T15:05:40.008372+0000","last_clean":"2024-09-13T15:05:40.008372+0000","last_became_active":"2024-09-13T15:05:11.105182+0000","last_became_peered":"2024-09-13T15:05:11.105182+0000","last_unstale":"2024-09-13T15:05:40.008372+0000","last_undegraded":"2024-09-13T15:05:40.008372+0000","last_fullsized":"2024-09-13T15:05:40.008372+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2341,"log_dups_size":0,"ondisk_log_size":2341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:16.270466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2341,"num_write_kb":149824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":653,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.425075+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:05:23.425075+0000","last_peered":"2024-09-13T15:05:23.425075+0000","last_clean":"2024-09-13T15:05:23.425075+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:05:23.425075+0000","last_undegraded":"2024-09-13T15:05:23.425075+0000","last_fullsized":"2024-09-13T15:05:23.425075+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":210,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834333+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:05:39.834333+0000","last_peered":"2024-09-13T15:05:39.834333+0000","last_clean":"2024-09-13T15:05:39.834333+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:05:39.834333+0000","last_undegraded":"2024-09-13T15:05:39.834333+0000","last_fullsized":"2024-09-13T15:05:39.834333+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":713,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817590+0000","last_change":"2024-09-13T15:03:44.116244+0000","last_active":"2024-09-13T15:05:39.817590+0000","last_peered":"2024-09-13T15:05:39.817590+0000","last_clean":"2024-09-13T15:05:39.817590+0000","last_became_active":"2024-09-13T15:03:44.115926+0000","last_became_peered":"2024-09-13T15:03:44.115926+0000","last_unstale":"2024-09-13T15:05:39.817590+0000","last_undegraded":"2024-09-13T15:05:39.817590+0000","last_fullsized":"2024-09-13T15:05:39.817590+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:01.231884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":569,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.820510+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:05:39.820510+0000","last_peered":"2024-09-13T15:05:39.820510+0000","last_clean":"2024-09-13T15:05:39.820510+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:05:39.820510+0000","last_undegraded":"2024-09-13T15:05:39.820510+0000","last_fullsized":"2024-09-13T15:05:39.820510+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.9","version":"401'2657","reported_seq":2679,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:42.285116+0000","last_change":"2024-09-13T15:05:11.537327+0000","last_active":"2024-09-13T15:05:42.285116+0000","last_peered":"2024-09-13T15:05:42.285116+0000","last_clean":"2024-09-13T15:05:42.285116+0000","last_became_active":"2024-09-13T15:05:11.536882+0000","last_became_peered":"2024-09-13T15:05:11.536882+0000","last_unstale":"2024-09-13T15:05:42.285116+0000","last_undegraded":"2024-09-13T15:05:42.285116+0000","last_fullsized":"2024-09-13T15:05:42.285116+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2657,"log_dups_size":0,"ondisk_log_size":2657,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:14.728744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174129152,"num_objects":2657,"num_object_clones":0,"num_object_copies":7971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2657,"num_write_kb":170048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":719,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424226+0000","last_change":"2024-09-13T15:03:46.574886+0000","last_active":"2024-09-13T15:05:23.424226+0000","last_peered":"2024-09-13T15:05:23.424226+0000","last_clean":"2024-09-13T15:05:23.424226+0000","last_became_active":"2024-09-13T15:03:46.574563+0000","last_became_peered":"2024-09-13T15:03:46.574563+0000","last_unstale":"2024-09-13T15:05:23.424226+0000","last_undegraded":"2024-09-13T15:05:23.424226+0000","last_fullsized":"2024-09-13T15:05:23.424226+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:29.856598+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":630,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817878+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:05:39.817878+0000","last_peered":"2024-09-13T15:05:39.817878+0000","last_clean":"2024-09-13T15:05:39.817878+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:05:39.817878+0000","last_undegraded":"2024-09-13T15:05:39.817878+0000","last_fullsized":"2024-09-13T15:05:39.817878+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":192,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717213+0000","last_change":"2024-09-13T15:03:44.121114+0000","last_active":"2024-09-13T15:05:22.717213+0000","last_peered":"2024-09-13T15:05:22.717213+0000","last_clean":"2024-09-13T15:05:22.717213+0000","last_became_active":"2024-09-13T15:03:44.120758+0000","last_became_peered":"2024-09-13T15:03:44.120758+0000","last_unstale":"2024-09-13T15:05:22.717213+0000","last_undegraded":"2024-09-13T15:05:22.717213+0000","last_fullsized":"2024-09-13T15:05:22.717213+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:54.923156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":68,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.818004+0000","last_change":"2024-09-13T15:03:44.114482+0000","last_active":"2024-09-13T15:05:39.818004+0000","last_peered":"2024-09-13T15:05:39.818004+0000","last_clean":"2024-09-13T15:05:39.818004+0000","last_became_active":"2024-09-13T15:03:44.112887+0000","last_became_peered":"2024-09-13T15:03:44.112887+0000","last_unstale":"2024-09-13T15:05:39.818004+0000","last_undegraded":"2024-09-13T15:05:39.818004+0000","last_fullsized":"2024-09-13T15:05:39.818004+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:37.719076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.a","version":"401'2432","reported_seq":2455,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.984955+0000","last_change":"2024-09-13T15:05:11.537005+0000","last_active":"2024-09-13T15:05:39.984955+0000","last_peered":"2024-09-13T15:05:39.984955+0000","last_clean":"2024-09-13T15:05:39.984955+0000","last_became_active":"2024-09-13T15:05:11.536634+0000","last_became_peered":"2024-09-13T15:05:11.536634+0000","last_unstale":"2024-09-13T15:05:39.984955+0000","last_undegraded":"2024-09-13T15:05:39.984955+0000","last_fullsized":"2024-09-13T15:05:39.984955+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2432,"log_dups_size":0,"ondisk_log_size":2432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:37.713772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159383552,"num_objects":2432,"num_object_clones":0,"num_object_copies":7296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2432,"num_write_kb":155648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":582,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:31.482633+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:05:31.482633+0000","last_peered":"2024-09-13T15:05:31.482633+0000","last_clean":"2024-09-13T15:05:31.482633+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:05:31.482633+0000","last_undegraded":"2024-09-13T15:05:31.482633+0000","last_fullsized":"2024-09-13T15:05:31.482633+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":662,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834324+0000","last_change":"2024-09-13T15:03:44.131836+0000","last_active":"2024-09-13T15:05:39.834324+0000","last_peered":"2024-09-13T15:05:39.834324+0000","last_clean":"2024-09-13T15:05:39.834324+0000","last_became_active":"2024-09-13T15:03:44.120633+0000","last_became_peered":"2024-09-13T15:03:44.120633+0000","last_unstale":"2024-09-13T15:05:39.834324+0000","last_undegraded":"2024-09-13T15:05:39.834324+0000","last_fullsized":"2024-09-13T15:05:39.834324+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:52.065604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":644,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817074+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:05:39.817074+0000","last_peered":"2024-09-13T15:05:39.817074+0000","last_clean":"2024-09-13T15:05:39.817074+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:05:39.817074+0000","last_undegraded":"2024-09-13T15:05:39.817074+0000","last_fullsized":"2024-09-13T15:05:39.817074+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":662,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833309+0000","last_change":"2024-09-13T15:03:44.123687+0000","last_active":"2024-09-13T15:05:39.833309+0000","last_peered":"2024-09-13T15:05:39.833309+0000","last_clean":"2024-09-13T15:05:39.833309+0000","last_became_active":"2024-09-13T15:03:44.123135+0000","last_became_peered":"2024-09-13T15:03:44.123135+0000","last_unstale":"2024-09-13T15:05:39.833309+0000","last_undegraded":"2024-09-13T15:05:39.833309+0000","last_fullsized":"2024-09-13T15:05:39.833309+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:42.101656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"400'2004","reported_seq":2023,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:37.799295+0000","last_change":"2024-09-13T15:05:11.104176+0000","last_active":"2024-09-13T15:05:37.799295+0000","last_peered":"2024-09-13T15:05:37.799295+0000","last_clean":"2024-09-13T15:05:37.799295+0000","last_became_active":"2024-09-13T15:05:11.103879+0000","last_became_peered":"2024-09-13T15:05:11.103879+0000","last_unstale":"2024-09-13T15:05:37.799295+0000","last_undegraded":"2024-09-13T15:05:37.799295+0000","last_fullsized":"2024-09-13T15:05:37.799295+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":2004,"log_dups_size":0,"ondisk_log_size":2004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:43:06.183789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2004,"num_write_kb":128256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":891,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.836051+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:05:39.836051+0000","last_peered":"2024-09-13T15:05:39.836051+0000","last_clean":"2024-09-13T15:05:39.836051+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:05:39.836051+0000","last_undegraded":"2024-09-13T15:05:39.836051+0000","last_fullsized":"2024-09-13T15:05:39.836051+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":423,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.816799+0000","last_change":"2024-09-13T15:03:44.127913+0000","last_active":"2024-09-13T15:05:39.816799+0000","last_peered":"2024-09-13T15:05:39.816799+0000","last_clean":"2024-09-13T15:05:39.816799+0000","last_became_active":"2024-09-13T15:03:44.121626+0000","last_became_peered":"2024-09-13T15:03:44.121626+0000","last_unstale":"2024-09-13T15:05:39.816799+0000","last_undegraded":"2024-09-13T15:05:39.816799+0000","last_fullsized":"2024-09-13T15:05:39.816799+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:24.441513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":390,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424279+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:05:23.424279+0000","last_peered":"2024-09-13T15:05:23.424279+0000","last_clean":"2024-09-13T15:05:23.424279+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:05:23.424279+0000","last_undegraded":"2024-09-13T15:05:23.424279+0000","last_fullsized":"2024-09-13T15:05:23.424279+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":89,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:31.482497+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:05:31.482497+0000","last_peered":"2024-09-13T15:05:31.482497+0000","last_clean":"2024-09-13T15:05:31.482497+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:05:31.482497+0000","last_undegraded":"2024-09-13T15:05:31.482497+0000","last_fullsized":"2024-09-13T15:05:31.482497+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717074+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T15:05:22.717074+0000","last_peered":"2024-09-13T15:05:22.717074+0000","last_clean":"2024-09-13T15:05:22.717074+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T15:05:22.717074+0000","last_undegraded":"2024-09-13T15:05:22.717074+0000","last_fullsized":"2024-09-13T15:05:22.717074+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":98,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716941+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:05:22.716941+0000","last_peered":"2024-09-13T15:05:22.716941+0000","last_clean":"2024-09-13T15:05:22.716941+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:05:22.716941+0000","last_undegraded":"2024-09-13T15:05:22.716941+0000","last_fullsized":"2024-09-13T15:05:22.716941+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":550,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.820769+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:05:39.820769+0000","last_peered":"2024-09-13T15:05:39.820769+0000","last_clean":"2024-09-13T15:05:39.820769+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:05:39.820769+0000","last_undegraded":"2024-09-13T15:05:39.820769+0000","last_fullsized":"2024-09-13T15:05:39.820769+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":603,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.538912+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:05:22.538912+0000","last_peered":"2024-09-13T15:05:22.538912+0000","last_clean":"2024-09-13T15:05:22.538912+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:05:22.538912+0000","last_undegraded":"2024-09-13T15:05:22.538912+0000","last_fullsized":"2024-09-13T15:05:22.538912+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":169,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.817247+0000","last_change":"2024-09-13T15:01:33.853899+0000","last_active":"2024-09-13T15:05:39.817247+0000","last_peered":"2024-09-13T15:05:39.817247+0000","last_clean":"2024-09-13T15:05:39.817247+0000","last_became_active":"2024-09-13T15:01:33.853515+0000","last_became_peered":"2024-09-13T15:01:33.853515+0000","last_unstale":"2024-09-13T15:05:39.817247+0000","last_undegraded":"2024-09-13T15:05:39.817247+0000","last_fullsized":"2024-09-13T15:05:39.817247+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:04.035329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.424383+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:05:23.424383+0000","last_peered":"2024-09-13T15:05:23.424383+0000","last_clean":"2024-09-13T15:05:23.424383+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:05:23.424383+0000","last_undegraded":"2024-09-13T15:05:23.424383+0000","last_fullsized":"2024-09-13T15:05:23.424383+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":257,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834169+0000","last_change":"2024-09-13T15:03:44.113646+0000","last_active":"2024-09-13T15:05:39.834169+0000","last_peered":"2024-09-13T15:05:39.834169+0000","last_clean":"2024-09-13T15:05:39.834169+0000","last_became_active":"2024-09-13T15:03:44.113342+0000","last_became_peered":"2024-09-13T15:03:44.113342+0000","last_unstale":"2024-09-13T15:05:39.834169+0000","last_undegraded":"2024-09-13T15:05:39.834169+0000","last_fullsized":"2024-09-13T15:05:39.834169+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:29.444600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":629,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833213+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:05:39.833213+0000","last_peered":"2024-09-13T15:05:39.833213+0000","last_clean":"2024-09-13T15:05:39.833213+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:05:39.833213+0000","last_undegraded":"2024-09-13T15:05:39.833213+0000","last_fullsized":"2024-09-13T15:05:39.833213+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":377,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:23.425124+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T15:05:23.425124+0000","last_peered":"2024-09-13T15:05:23.425124+0000","last_clean":"2024-09-13T15:05:23.425124+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T15:05:23.425124+0000","last_undegraded":"2024-09-13T15:05:23.425124+0000","last_fullsized":"2024-09-13T15:05:23.425124+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":691,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833406+0000","last_change":"2024-09-13T15:03:44.123370+0000","last_active":"2024-09-13T15:05:39.833406+0000","last_peered":"2024-09-13T15:05:39.833406+0000","last_clean":"2024-09-13T15:05:39.833406+0000","last_became_active":"2024-09-13T15:03:44.122679+0000","last_became_peered":"2024-09-13T15:03:44.122679+0000","last_unstale":"2024-09-13T15:05:39.833406+0000","last_undegraded":"2024-09-13T15:05:39.833406+0000","last_fullsized":"2024-09-13T15:05:39.833406+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:24.806181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":399,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833781+0000","last_change":"2024-09-13T15:03:44.127512+0000","last_active":"2024-09-13T15:05:39.833781+0000","last_peered":"2024-09-13T15:05:39.833781+0000","last_clean":"2024-09-13T15:05:39.833781+0000","last_became_active":"2024-09-13T15:03:44.126735+0000","last_became_peered":"2024-09-13T15:03:44.126735+0000","last_unstale":"2024-09-13T15:05:39.833781+0000","last_undegraded":"2024-09-13T15:05:39.833781+0000","last_fullsized":"2024-09-13T15:05:39.833781+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:20.140202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":642,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.815283+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:05:39.815283+0000","last_peered":"2024-09-13T15:05:39.815283+0000","last_clean":"2024-09-13T15:05:39.815283+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:05:39.815283+0000","last_undegraded":"2024-09-13T15:05:39.815283+0000","last_fullsized":"2024-09-13T15:05:39.815283+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":643,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.834125+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:05:39.834125+0000","last_peered":"2024-09-13T15:05:39.834125+0000","last_clean":"2024-09-13T15:05:39.834125+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:05:39.834125+0000","last_undegraded":"2024-09-13T15:05:39.834125+0000","last_fullsized":"2024-09-13T15:05:39.834125+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716709+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T15:05:22.716709+0000","last_peered":"2024-09-13T15:05:22.716709+0000","last_clean":"2024-09-13T15:05:22.716709+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T15:05:22.716709+0000","last_undegraded":"2024-09-13T15:05:22.716709+0000","last_fullsized":"2024-09-13T15:05:22.716709+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:59.608331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":653,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539409+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:05:22.539409+0000","last_peered":"2024-09-13T15:05:22.539409+0000","last_clean":"2024-09-13T15:05:22.539409+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:05:22.539409+0000","last_undegraded":"2024-09-13T15:05:22.539409+0000","last_fullsized":"2024-09-13T15:05:22.539409+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":74,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716880+0000","last_change":"2024-09-13T15:03:44.124264+0000","last_active":"2024-09-13T15:05:22.716880+0000","last_peered":"2024-09-13T15:05:22.716880+0000","last_clean":"2024-09-13T15:05:22.716880+0000","last_became_active":"2024-09-13T15:03:44.123914+0000","last_became_peered":"2024-09-13T15:03:44.123914+0000","last_unstale":"2024-09-13T15:05:22.716880+0000","last_undegraded":"2024-09-13T15:05:22.716880+0000","last_fullsized":"2024-09-13T15:05:22.716880+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:55.477300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716784+0000","last_change":"2024-09-13T15:04:24.852045+0000","last_active":"2024-09-13T15:05:22.716784+0000","last_peered":"2024-09-13T15:05:22.716784+0000","last_clean":"2024-09-13T15:05:22.716784+0000","last_became_active":"2024-09-13T15:04:24.851772+0000","last_became_peered":"2024-09-13T15:04:24.851772+0000","last_unstale":"2024-09-13T15:05:22.716784+0000","last_undegraded":"2024-09-13T15:05:22.716784+0000","last_fullsized":"2024-09-13T15:05:22.716784+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:53:57.699649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":370,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716896+0000","last_change":"2024-09-13T15:01:33.852507+0000","last_active":"2024-09-13T15:05:22.716896+0000","last_peered":"2024-09-13T15:05:22.716896+0000","last_clean":"2024-09-13T15:05:22.716896+0000","last_became_active":"2024-09-13T15:01:33.852196+0000","last_became_peered":"2024-09-13T15:01:33.852196+0000","last_unstale":"2024-09-13T15:05:22.716896+0000","last_undegraded":"2024-09-13T15:05:22.716896+0000","last_fullsized":"2024-09-13T15:05:22.716896+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:28.217977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":108,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716607+0000","last_change":"2024-09-13T15:03:44.116627+0000","last_active":"2024-09-13T15:05:22.716607+0000","last_peered":"2024-09-13T15:05:22.716607+0000","last_clean":"2024-09-13T15:05:22.716607+0000","last_became_active":"2024-09-13T15:03:44.116197+0000","last_became_peered":"2024-09-13T15:03:44.116197+0000","last_unstale":"2024-09-13T15:05:22.716607+0000","last_undegraded":"2024-09-13T15:05:22.716607+0000","last_fullsized":"2024-09-13T15:05:22.716607+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:30.519779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":355,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.717127+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:05:22.717127+0000","last_peered":"2024-09-13T15:05:22.717127+0000","last_clean":"2024-09-13T15:05:22.717127+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:05:22.717127+0000","last_undegraded":"2024-09-13T15:05:22.717127+0000","last_fullsized":"2024-09-13T15:05:22.717127+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":693,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.835683+0000","last_change":"2024-09-13T15:03:44.120794+0000","last_active":"2024-09-13T15:05:39.835683+0000","last_peered":"2024-09-13T15:05:39.835683+0000","last_clean":"2024-09-13T15:05:39.835683+0000","last_became_active":"2024-09-13T15:03:44.120431+0000","last_became_peered":"2024-09-13T15:03:44.120431+0000","last_unstale":"2024-09-13T15:05:39.835683+0000","last_undegraded":"2024-09-13T15:05:39.835683+0000","last_fullsized":"2024-09-13T15:05:39.835683+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:27.396753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":652,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.833961+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:05:39.833961+0000","last_peered":"2024-09-13T15:05:39.833961+0000","last_clean":"2024-09-13T15:05:39.833961+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:05:39.833961+0000","last_undegraded":"2024-09-13T15:05:39.833961+0000","last_fullsized":"2024-09-13T15:05:39.833961+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.716517+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T15:05:22.716517+0000","last_peered":"2024-09-13T15:05:22.716517+0000","last_clean":"2024-09-13T15:05:22.716517+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T15:05:22.716517+0000","last_undegraded":"2024-09-13T15:05:22.716517+0000","last_fullsized":"2024-09-13T15:05:22.716517+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":85,"reported_epoch":400,"state":"active+clean","last_fresh":"2024-09-13T15:05:22.539576+0000","last_change":"2024-09-13T15:03:44.120052+0000","last_active":"2024-09-13T15:05:22.539576+0000","last_peered":"2024-09-13T15:05:22.539576+0000","last_clean":"2024-09-13T15:05:22.539576+0000","last_became_active":"2024-09-13T15:03:44.109611+0000","last_became_peered":"2024-09-13T15:03:44.109611+0000","last_unstale":"2024-09-13T15:05:22.539576+0000","last_undegraded":"2024-09-13T15:05:22.539576+0000","last_fullsized":"2024-09-13T15:05:22.539576+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:04:55.196794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":2567700480,"num_objects":39180,"num_object_clones":0,"num_object_copies":117540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39180,"num_write_kb":2507520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7704543232,"data_stored":7704543232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39180,"ondisk_log_size":39180,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234547,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957128,"kb_used_data":628352,"kb_used_omap":1,"kb_used_meta":1328766,"kb_avail":91788024,"statfs":{"total":95995035648,"available":93990936576,"internally_reserved":0,"allocated":643432448,"data_stored":641262010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1360656843},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496185,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2343628,"kb_used_data":1060864,"kb_used_omap":1,"kb_used_meta":1282750,"kb_avail":91401524,"statfs":{"total":95995035648,"available":93595160576,"internally_reserved":0,"allocated":1086324736,"data_stored":1084154843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1313536455},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757823,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2176960,"kb_used_data":783988,"kb_used_omap":1,"kb_used_meta":1392958,"kb_avail":91568192,"statfs":{"total":95995035648,"available":93765828608,"internally_reserved":0,"allocated":802803712,"data_stored":800641564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1426389446},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019459,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262920,"kb_used_data":1090688,"kb_used_omap":1,"kb_used_meta":1172222,"kb_avail":91482232,"statfs":{"total":95995035648,"available":93677805568,"internally_reserved":0,"allocated":1116864512,"data_stored":1114694094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1200355782},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650671,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2595388,"kb_used_data":1649072,"kb_used_omap":4,"kb_used_meta":946299,"kb_avail":91149764,"statfs":{"total":95995035648,"available":93337358336,"internally_reserved":0,"allocated":1688649728,"data_stored":1686491438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":969010735},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879605,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1861876,"kb_used_data":916140,"kb_used_omap":6,"kb_used_meta":945721,"kb_avail":91883276,"statfs":{"total":95995035648,"available":94088474624,"internally_reserved":0,"allocated":938127360,"data_stored":935965406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":968418873},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435622,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2319716,"kb_used_data":1166492,"kb_used_omap":5,"kb_used_meta":1153210,"kb_avail":91425436,"statfs":{"total":95995035648,"available":93619646464,"internally_reserved":0,"allocated":1194487808,"data_stored":1192340483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1180887393},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":1494648619066,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1245118464,"data_stored":1245118464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":954859520,"data_stored":954859520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1737228288,"data_stored":1737228288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1157758976,"data_stored":1157758976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":822738944,"data_stored":822738944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1135607808,"data_stored":1135607808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":651231232,"data_stored":651231232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:05:45.397 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:05:45.398 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:05:45.398 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_9 2024-09-13T15:05:45.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:05:45.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:45.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 21 16 47685 47669 141.856 107.5 0.0172411 0.00704327 2024-09-13T15:05:45.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:45.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:45.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:45.675 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:45.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:45.777 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:45.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:45.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:05:45.860 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:05:45.861 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:05:45.862 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:05:45.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:45.960 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:45.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:45.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:46.062 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:46.096 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:46.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:45 smithi042 ceph-mon[113926]: pgmap v1441: 99 pgs: 99 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:05:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:45 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3033521165' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:46.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:45 smithi042 ceph-mon[108570]: pgmap v1441: 99 pgs: 99 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:05:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:45 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3033521165' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:46.198 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:46.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:45 smithi159 ceph-mon[102519]: pgmap v1441: 99 pgs: 99 active+clean; 2.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 126 MiB/s wr, 2.02k op/s 2024-09-13T15:05:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:45 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3033521165' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:05:46.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:46.372 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:46.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:46.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 22 16 49406 49390 140.297 107.562 0.00665469 0.00712246 2024-09-13T15:05:46.474 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:46.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:46.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:46.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:46.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.643 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:46.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:46.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:46.785 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:46.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:46.888 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:46.908 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:46.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:46.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:46.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:46.966 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:05:46.966 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:05:46.966 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:05:46.966 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:05:46.966 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:05:47.016 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:47.053 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:47.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:47.156 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:47.189 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:47.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:47.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:47.320 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:47.424 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:47.463 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 23 16 51083 51067 138.751 104.812 0.0102476 0.00720104 2024-09-13T15:05:47.465 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:47.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:47.598 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:47.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:47.700 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:47.734 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:47.836 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:47.874 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:47.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:47.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:48.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:48.094 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:47 smithi042 ceph-mon[108570]: pgmap v1442: 99 pgs: 99 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:05:48.095 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:47 smithi042 ceph-mon[113926]: pgmap v1442: 99 pgs: 99 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:05:48.109 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:48.134 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:05:48.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:48.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:47 smithi159 ceph-mon[102519]: pgmap v1442: 99 pgs: 99 active+clean; 2.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 120 MiB/s wr, 1.92k op/s 2024-09-13T15:05:48.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:48.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:48.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:48.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:48.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:48.465 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 24 16 52771 52755 137.357 105.5 0.00856708 0.00727505 2024-09-13T15:05:48.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:48.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:48.632 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:48.669 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:05:48.675 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:48.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:48.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:48.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:48.889 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:48.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:48.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:49.031 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:49.133 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:49.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:49.306 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:49.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:49.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:49.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 25 16 54362 54346 135.837 99.4375 0.00831591 0.0073564 2024-09-13T15:05:49.497 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:49.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:49.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:49.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.639 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:49.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:49.740 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:49.766 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:49.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:49.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:49.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:49.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:49.971 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:49.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:50.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.083 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:49 smithi042 ceph-mon[108570]: pgmap v1443: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T15:05:50.084 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:49] "GET /metrics HTTP/1.1" 200 36523 "" "Prometheus/2.51.0" 2024-09-13T15:05:50.084 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:49 smithi042 ceph-mon[113926]: pgmap v1443: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T15:05:50.149 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:50.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:50.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:50.203 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:05:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:49 smithi159 ceph-mon[102519]: pgmap v1443: 99 pgs: 99 active+clean; 3.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-13T15:05:50.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:50.330 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:50.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:50.431 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:50.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 26 16 55824 55808 134.126 91.375 0.0132765 0.0074502 2024-09-13T15:05:50.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:50.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:50.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:50.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:50.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:50.709 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:50.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:50.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:50.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:50.869 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:50.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:50.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:50.897 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:05:50.897 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4100677744' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:50.898 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:50.898 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:50.898 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4100677744' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:50.898 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:50.898 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:50.971 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:51.007 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:51.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:51.152 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:51.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:50 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4100677744' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:50 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:05:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:05:51.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.286 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:51.387 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:51.418 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:51.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 27 16 57346 57330 132.682 95.125 0.0114667 0.00753169 2024-09-13T15:05:51.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:51.553 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:51.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:51.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:51.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:51.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:51.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:51.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:51.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:51.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:52.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:52.101 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:52.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:52.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:51 smithi159 ceph-mon[102519]: pgmap v1444: 99 pgs: 99 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-09-13T15:05:52.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:51 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:52.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:51 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:05:52.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:51 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:52.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:51 smithi159 ceph-mon[102519]: osdmap e402: 8 total, 8 up, 7 in 2024-09-13T15:05:52.180 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[108570]: pgmap v1444: 99 pgs: 99 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-09-13T15:05:52.180 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:52.180 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:05:52.180 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:52.180 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[108570]: osdmap e402: 8 total, 8 up, 7 in 2024-09-13T15:05:52.181 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[113926]: pgmap v1444: 99 pgs: 99 active+clean; 3.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 112 MiB/s wr, 1.78k op/s 2024-09-13T15:05:52.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:05:52.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:05:52.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:05:52.182 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:51 smithi042 ceph-mon[113926]: osdmap e402: 8 total, 8 up, 7 in 2024-09-13T15:05:52.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:52.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:52.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:52.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:52.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:52.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 28 16 58940 58924 131.501 99.625 0.0130112 0.0075992 2024-09-13T15:05:52.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:52.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:52.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:52.670 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:52.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:52.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.803 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:52.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:52.904 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:52.945 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:52.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:53.046 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:53.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:53.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:53.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:53.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:53.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:53.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:53.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:53.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 29 16 60544 60528 130.423 100.25 0.00796548 0.00766208 2024-09-13T15:05:53.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:53.591 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:53.630 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:53.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:53.732 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:53.770 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:53.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:53.871 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:53.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:53.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:54.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:54.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:54.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:54.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:54.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:53 smithi159 ceph-mon[102519]: pgmap v1446: 99 pgs: 99 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:05:54.274 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:53 smithi042 ceph-mon[113926]: pgmap v1446: 99 pgs: 99 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:05:54.274 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:53 smithi042 ceph-mon[108570]: pgmap v1446: 99 pgs: 99 active+clean; 3.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-13T15:05:54.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:54.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.324 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:54.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:54.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:54.463 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:54.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 30 16 61972 61956 129.051 89.25 0.0121114 0.00774393 2024-09-13T15:05:54.565 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:54.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:54.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:54.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:54.839 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:54.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:54.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:54.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:54.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:55.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:55.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:55.069 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:05:55.106 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:55.166 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:55.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:55.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:55.313 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:55.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:55.416 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:55.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 31 16 63217 63201 127.398 77.8125 0.00323878 0.00784266 2024-09-13T15:05:55.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.470 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:55.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:55.571 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:55.599 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:55.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:55.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:55.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:55.875 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:55.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:55.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:56.011 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:56.113 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:56.149 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:56.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:55 smithi159 ceph-mon[102519]: pgmap v1447: 99 pgs: 99 active+clean; 3.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-13T15:05:56.250 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.280 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:56.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:55 smithi042 ceph-mon[113926]: pgmap v1447: 99 pgs: 99 active+clean; 3.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-13T15:05:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:55 smithi042 ceph-mon[108570]: pgmap v1447: 99 pgs: 99 active+clean; 3.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-13T15:05:56.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:56.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:56.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:56.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 32 16 64546 64530 126.012 83.0625 0.0129972 0.00793095 2024-09-13T15:05:56.525 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:56.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:56.683 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:56.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:56.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:56.918 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:56.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:56.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:56.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:57.053 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:57.092 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:57.194 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:57.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:57.324 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:57.359 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:57.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:57.466 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 33 16 65912 65896 124.78 85.375 0.0193166 0.00800835 2024-09-13T15:05:57.492 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:57.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:57.594 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:57.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:57.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:57.745 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:57.790 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:57.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:57.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:57.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:57.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:58.024 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:58.066 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:58.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:05:58.212 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:58.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:05:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:57 smithi159 ceph-mon[102519]: pgmap v1448: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-13T15:05:58.314 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:58.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:58.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:58.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:57 smithi042 ceph-mon[113926]: pgmap v1448: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-13T15:05:58.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:57 smithi042 ceph-mon[108570]: pgmap v1448: 99 pgs: 99 active+clean; 3.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-13T15:05:58.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:58.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 34 16 67243 67227 123.555 83.1875 0.0119238 0.00808884 2024-09-13T15:05:58.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.479 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:58.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:58.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:58.603 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:58.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:05:58.705 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:58.743 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:58.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:58.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:05:58.882 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:58.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:58.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:05:58.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:59.018 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:59.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:59.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:59.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:05:59.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:59.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:05:59.381 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:05:59.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:05:59.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 35 16 68594 68578 122.438 84.4375 0.00480794 0.00816222 2024-09-13T15:05:59.512 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:05:59.535 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:05:59.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:05:59.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:05:59.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:05:59.796 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:05:59.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:05:59.898 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:05:59.937 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:05:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:05:59.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:00.039 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:00.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:05:59 smithi042 ceph-mon[108570]: pgmap v1449: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:06:00.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:05:59 smithi042 ceph-mon[113926]: pgmap v1449: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:06:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:05:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:05:59] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.51.0" 2024-09-13T15:06:00.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:00.209 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:00.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:05:59 smithi159 ceph-mon[102519]: pgmap v1449: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s 2024-09-13T15:06:00.310 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:00.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:00.450 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:00.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 36 16 69885 69869 121.278 80.6875 0.0135722 0.00823602 2024-09-13T15:06:00.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:00.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:00.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:00.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:00.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:00.708 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:00.738 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:00.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:00.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:00.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:00.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:00.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:01.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:01.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.157 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:01.258 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:01.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.300 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:01.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:01.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:01.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:01.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 37 16 71210 71194 120.238 82.8125 0.0111056 0.00831187 2024-09-13T15:06:01.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.578 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:01.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:01.679 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:01.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:01.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:01 smithi159 ceph-mon[102519]: pgmap v1450: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T15:06:01.818 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:01.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:01.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:01.958 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:01 smithi042 ceph-mon[108570]: pgmap v1450: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T15:06:01.959 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:01 smithi042 ceph-mon[113926]: pgmap v1450: 99 pgs: 99 active+clean; 4.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-09-13T15:06:01.981 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:01.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:01.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:02.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:02.117 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:02.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:02.266 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:02.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:02.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.405 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:02.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:02.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 38 16 72538 72522 119.258 83 0.0181258 0.00838004 2024-09-13T15:06:02.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:02.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:02.640 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:02.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:02.781 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:02.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:02.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:02.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:02.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:02.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:03.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:03.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:03.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:03.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:03.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:03.327 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:03.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:03.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:03.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:03.467 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 39 16 73798 73782 118.219 78.75 0.0175554 0.00845237 2024-09-13T15:06:03.501 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:03.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:03.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:03.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:03.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:03.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:03.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:03.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:03.922 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:03.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:03.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:04.024 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.064 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:04.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:03 smithi042 ceph-mon[108570]: pgmap v1451: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T15:06:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:03 smithi042 ceph-mon[113926]: pgmap v1451: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T15:06:04.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:04.204 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:04.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:03 smithi159 ceph-mon[102519]: pgmap v1451: 99 pgs: 99 active+clean; 4.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-09-13T15:06:04.305 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:04.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:04.438 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:04.459 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:04.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:06:04.467462+0000 min lat: 0.00110032 max lat: 0.202872 avg lat: 0.00850918 2024-09-13T15:06:04.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:06:04.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 40 16 75099 75083 117.296 81.3125 0.0150785 0.00850918 2024-09-13T15:06:04.561 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:04.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:04.685 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:04.707 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:04.809 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:04.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:04.939 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:04.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:04.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:04.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:05.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:05.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:05.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:05.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:05.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.255 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:05.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:05.356 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:05.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:05.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 41 16 75974 75958 115.769 54.6875 0.00979461 0.00863312 2024-09-13T15:06:05.493 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:05.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:05.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:05.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:05.784 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:05.814 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:05.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:05.916 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:05.952 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:05.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:05.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.054 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:06.094 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:06.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:05 smithi042 ceph-mon[108570]: pgmap v1452: 99 pgs: 99 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T15:06:06.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:05 smithi042 ceph-mon[113926]: pgmap v1452: 99 pgs: 99 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T15:06:06.196 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:06.234 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:06.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:05 smithi159 ceph-mon[102519]: pgmap v1452: 99 pgs: 99 active+clean; 4.4 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-13T15:06:06.273 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:06:06.275 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:06:06.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:06.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:06.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 42 16 77187 77171 114.818 75.8125 0.0215061 0.00870456 2024-09-13T15:06:06.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.507 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:06.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.608 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:06.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.637 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:06.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:06.738 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:06.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:06.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:06.915 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:06.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:07.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:07.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.060 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:07.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:07.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[108570]: from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:06 smithi042 ceph-mon[113926]: from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:07.162 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:07.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:06 smithi159 ceph-mon[102519]: from='client.46160 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:06:07.288 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:07.326 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:07.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:07.428 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:07.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:07.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 43 16 78484 78468 114.033 81.0625 0.0080329 0.00876382 2024-09-13T15:06:07.566 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:07.607 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:07.709 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:07.744 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:07.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:07.759 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1452,"stamp":"2024-09-13T15:06:04.725689+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4714403813,"num_objects":71953,"num_object_clones":0,"num_object_copies":215859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71991,"num_write_kb":4603361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73625,"ondisk_log_size":73625,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22123088,"kb_used_data":13677784,"kb_used_omap":22,"kb_used_meta":8445225,"kb_avail":634092976,"statfs":{"total":671965249536,"available":649311207424,"internally_reserved":0,"allocated":14006050816,"data_stored":13990875142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":8647910631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,9,6,6,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1023016960,"num_objects":15610,"num_object_clones":0,"num_object_copies":46830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15610,"num_write_kb":999040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003801"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":631,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.897921+0000","last_change":"2024-09-13T14:40:37.181148+0000","last_active":"2024-09-13T15:05:50.897921+0000","last_peered":"2024-09-13T15:05:50.897921+0000","last_clean":"2024-09-13T15:05:50.897921+0000","last_became_active":"2024-09-13T14:40:37.180836+0000","last_became_peered":"2024-09-13T14:40:37.180836+0000","last_unstale":"2024-09-13T15:05:50.897921+0000","last_undegraded":"2024-09-13T15:05:50.897921+0000","last_fullsized":"2024-09-13T15:05:50.897921+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:24:47.973601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":95,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826733+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:05:39.826733+0000","last_peered":"2024-09-13T15:05:39.826733+0000","last_clean":"2024-09-13T15:05:39.826733+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:05:39.826733+0000","last_undegraded":"2024-09-13T15:05:39.826733+0000","last_fullsized":"2024-09-13T15:05:39.826733+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901842+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:05:50.901842+0000","last_peered":"2024-09-13T15:05:50.901842+0000","last_clean":"2024-09-13T15:05:50.901842+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:05:50.901842+0000","last_undegraded":"2024-09-13T15:05:50.901842+0000","last_fullsized":"2024-09-13T15:05:50.901842+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":400,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.900604+0000","last_change":"2024-09-13T15:03:44.123745+0000","last_active":"2024-09-13T15:05:50.900604+0000","last_peered":"2024-09-13T15:05:50.900604+0000","last_clean":"2024-09-13T15:05:50.900604+0000","last_became_active":"2024-09-13T15:03:44.123397+0000","last_became_peered":"2024-09-13T15:03:44.123397+0000","last_unstale":"2024-09-13T15:05:50.900604+0000","last_undegraded":"2024-09-13T15:05:50.900604+0000","last_fullsized":"2024-09-13T15:05:50.900604+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.079957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":401,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901722+0000","last_change":"2024-09-13T15:03:44.128445+0000","last_active":"2024-09-13T15:05:50.901722+0000","last_peered":"2024-09-13T15:05:50.901722+0000","last_clean":"2024-09-13T15:05:50.901722+0000","last_became_active":"2024-09-13T15:03:44.128047+0000","last_became_peered":"2024-09-13T15:03:44.128047+0000","last_unstale":"2024-09-13T15:05:50.901722+0000","last_undegraded":"2024-09-13T15:05:50.901722+0000","last_fullsized":"2024-09-13T15:05:50.901722+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:22.359631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":89,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826089+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:05:39.826089+0000","last_peered":"2024-09-13T15:05:39.826089+0000","last_clean":"2024-09-13T15:05:39.826089+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:05:39.826089+0000","last_undegraded":"2024-09-13T15:05:39.826089+0000","last_fullsized":"2024-09-13T15:05:39.826089+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":657,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.898103+0000","last_change":"2024-09-13T14:40:37.180496+0000","last_active":"2024-09-13T15:05:50.898103+0000","last_peered":"2024-09-13T15:05:50.898103+0000","last_clean":"2024-09-13T15:05:50.898103+0000","last_became_active":"2024-09-13T14:40:37.180120+0000","last_became_peered":"2024-09-13T14:40:37.180120+0000","last_unstale":"2024-09-13T15:05:50.898103+0000","last_undegraded":"2024-09-13T15:05:50.898103+0000","last_fullsized":"2024-09-13T15:05:50.898103+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:04:32.564721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":495,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827158+0000","last_change":"2024-09-13T15:03:44.130129+0000","last_active":"2024-09-13T15:05:39.827158+0000","last_peered":"2024-09-13T15:05:39.827158+0000","last_clean":"2024-09-13T15:05:39.827158+0000","last_became_active":"2024-09-13T15:03:44.114883+0000","last_became_peered":"2024-09-13T15:03:44.114883+0000","last_unstale":"2024-09-13T15:05:39.827158+0000","last_undegraded":"2024-09-13T15:05:39.827158+0000","last_fullsized":"2024-09-13T15:05:39.827158+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:54.544220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"402'4486","reported_seq":4502,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.434434+0000","last_change":"2024-09-13T15:05:11.104668+0000","last_active":"2024-09-13T15:06:01.434434+0000","last_peered":"2024-09-13T15:06:01.434434+0000","last_clean":"2024-09-13T15:06:01.434434+0000","last_became_active":"2024-09-13T15:05:11.104349+0000","last_became_peered":"2024-09-13T15:05:11.104349+0000","last_unstale":"2024-09-13T15:06:01.434434+0000","last_undegraded":"2024-09-13T15:06:01.434434+0000","last_fullsized":"2024-09-13T15:06:01.434434+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4486,"log_dups_size":0,"ondisk_log_size":4486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:15:34.717367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293994496,"num_objects":4486,"num_object_clones":0,"num_object_copies":13458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4486,"num_write_kb":287104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":118,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901325+0000","last_change":"2024-09-13T15:03:44.129600+0000","last_active":"2024-09-13T15:05:50.901325+0000","last_peered":"2024-09-13T15:05:50.901325+0000","last_clean":"2024-09-13T15:05:50.901325+0000","last_became_active":"2024-09-13T15:03:44.123858+0000","last_became_peered":"2024-09-13T15:03:44.123858+0000","last_unstale":"2024-09-13T15:05:50.901325+0000","last_undegraded":"2024-09-13T15:05:50.901325+0000","last_fullsized":"2024-09-13T15:05:50.901325+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:40:15.345841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":61,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.891861+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:05:50.891861+0000","last_peered":"2024-09-13T15:05:50.891861+0000","last_clean":"2024-09-13T15:05:50.891861+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:05:50.891861+0000","last_undegraded":"2024-09-13T15:05:50.891861+0000","last_fullsized":"2024-09-13T15:05:50.891861+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"402'4583","reported_seq":4608,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.347415+0000","last_change":"2024-09-13T15:05:11.103932+0000","last_active":"2024-09-13T15:06:02.347415+0000","last_peered":"2024-09-13T15:06:02.347415+0000","last_clean":"2024-09-13T15:06:02.347415+0000","last_became_active":"2024-09-13T15:05:11.103484+0000","last_became_peered":"2024-09-13T15:05:11.103484+0000","last_unstale":"2024-09-13T15:06:02.347415+0000","last_undegraded":"2024-09-13T15:06:02.347415+0000","last_fullsized":"2024-09-13T15:06:02.347415+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4583,"log_dups_size":0,"ondisk_log_size":4583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:46:53.861057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300351488,"num_objects":4583,"num_object_clones":0,"num_object_copies":13749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4583,"num_write_kb":293312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":94,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.900845+0000","last_change":"2024-09-13T14:59:12.715977+0000","last_active":"2024-09-13T15:05:50.900845+0000","last_peered":"2024-09-13T15:05:50.900845+0000","last_clean":"2024-09-13T15:05:50.900845+0000","last_became_active":"2024-09-13T14:59:12.707728+0000","last_became_peered":"2024-09-13T14:59:12.707728+0000","last_unstale":"2024-09-13T15:05:50.900845+0000","last_undegraded":"2024-09-13T15:05:50.900845+0000","last_fullsized":"2024-09-13T15:05:50.900845+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:44:20.897906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":90,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:46.502159+0000","last_change":"2024-09-13T14:59:12.711847+0000","last_active":"2024-09-13T15:05:46.502159+0000","last_peered":"2024-09-13T15:05:46.502159+0000","last_clean":"2024-09-13T15:05:46.502159+0000","last_became_active":"2024-09-13T14:59:12.711192+0000","last_became_peered":"2024-09-13T14:59:12.711192+0000","last_unstale":"2024-09-13T15:05:46.502159+0000","last_undegraded":"2024-09-13T15:05:46.502159+0000","last_fullsized":"2024-09-13T15:05:46.502159+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:01.175642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"402'4468","reported_seq":4493,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.807491+0000","last_change":"2024-09-13T15:05:11.537086+0000","last_active":"2024-09-13T15:06:02.807491+0000","last_peered":"2024-09-13T15:06:02.807491+0000","last_clean":"2024-09-13T15:06:02.807491+0000","last_became_active":"2024-09-13T15:05:11.536730+0000","last_became_peered":"2024-09-13T15:05:11.536730+0000","last_unstale":"2024-09-13T15:06:02.807491+0000","last_undegraded":"2024-09-13T15:06:02.807491+0000","last_fullsized":"2024-09-13T15:06:02.807491+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:51.566063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292814848,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4468,"num_write_kb":285952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":72,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.828174+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:05:39.828174+0000","last_peered":"2024-09-13T15:05:39.828174+0000","last_clean":"2024-09-13T15:05:39.828174+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:05:39.828174+0000","last_undegraded":"2024-09-13T15:05:39.828174+0000","last_fullsized":"2024-09-13T15:05:39.828174+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":377,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827930+0000","last_change":"2024-09-13T14:35:58.856292+0000","last_active":"2024-09-13T15:05:39.827930+0000","last_peered":"2024-09-13T15:05:39.827930+0000","last_clean":"2024-09-13T15:05:39.827930+0000","last_became_active":"2024-09-13T14:35:58.855641+0000","last_became_peered":"2024-09-13T14:35:58.855641+0000","last_unstale":"2024-09-13T15:05:39.827930+0000","last_undegraded":"2024-09-13T15:05:39.827930+0000","last_fullsized":"2024-09-13T15:05:39.827930+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:36:33.906351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"402'4648","reported_seq":4673,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.306348+0000","last_change":"2024-09-13T15:05:11.101187+0000","last_active":"2024-09-13T15:06:02.306348+0000","last_peered":"2024-09-13T15:06:02.306348+0000","last_clean":"2024-09-13T15:06:02.306348+0000","last_became_active":"2024-09-13T15:05:11.100618+0000","last_became_peered":"2024-09-13T15:05:11.100618+0000","last_unstale":"2024-09-13T15:06:02.306348+0000","last_undegraded":"2024-09-13T15:06:02.306348+0000","last_fullsized":"2024-09-13T15:06:02.306348+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4648,"log_dups_size":0,"ondisk_log_size":4648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:51:13.595199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304611328,"num_objects":4648,"num_object_clones":0,"num_object_copies":13944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4648,"num_write_kb":297472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":196,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.898060+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:05:50.898060+0000","last_peered":"2024-09-13T15:05:50.898060+0000","last_clean":"2024-09-13T15:05:50.898060+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:05:50.898060+0000","last_undegraded":"2024-09-13T15:05:50.898060+0000","last_fullsized":"2024-09-13T15:05:50.898060+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":435,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893352+0000","last_change":"2024-09-13T15:03:44.122648+0000","last_active":"2024-09-13T15:05:50.893352+0000","last_peered":"2024-09-13T15:05:50.893352+0000","last_clean":"2024-09-13T15:05:50.893352+0000","last_became_active":"2024-09-13T15:03:44.121071+0000","last_became_peered":"2024-09-13T15:03:44.121071+0000","last_unstale":"2024-09-13T15:05:50.893352+0000","last_undegraded":"2024-09-13T15:05:50.893352+0000","last_fullsized":"2024-09-13T15:05:50.893352+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:39:51.385950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"402'4652","reported_seq":4677,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.766439+0000","last_change":"2024-09-13T15:05:11.103812+0000","last_active":"2024-09-13T15:06:02.766439+0000","last_peered":"2024-09-13T15:06:02.766439+0000","last_clean":"2024-09-13T15:06:02.766439+0000","last_became_active":"2024-09-13T15:05:11.103500+0000","last_became_peered":"2024-09-13T15:05:11.103500+0000","last_unstale":"2024-09-13T15:06:02.766439+0000","last_undegraded":"2024-09-13T15:06:02.766439+0000","last_fullsized":"2024-09-13T15:06:02.766439+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4652,"log_dups_size":0,"ondisk_log_size":4652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:31:33.535670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304873472,"num_objects":4652,"num_object_clones":0,"num_object_copies":13956,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4652,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4652,"num_write_kb":297728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":983,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.897657+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:05:50.897657+0000","last_peered":"2024-09-13T15:05:50.897657+0000","last_clean":"2024-09-13T15:05:50.897657+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:05:50.897657+0000","last_undegraded":"2024-09-13T15:05:50.897657+0000","last_fullsized":"2024-09-13T15:05:50.897657+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":706,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.897881+0000","last_change":"2024-09-13T15:03:44.133692+0000","last_active":"2024-09-13T15:05:50.897881+0000","last_peered":"2024-09-13T15:05:50.897881+0000","last_clean":"2024-09-13T15:05:50.897881+0000","last_became_active":"2024-09-13T15:03:44.125117+0000","last_became_peered":"2024-09-13T15:03:44.125117+0000","last_unstale":"2024-09-13T15:05:50.897881+0000","last_undegraded":"2024-09-13T15:05:50.897881+0000","last_fullsized":"2024-09-13T15:05:50.897881+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:50:12.857175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":90,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901283+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:05:50.901283+0000","last_peered":"2024-09-13T15:05:50.901283+0000","last_clean":"2024-09-13T15:05:50.901283+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:05:50.901283+0000","last_undegraded":"2024-09-13T15:05:50.901283+0000","last_fullsized":"2024-09-13T15:05:50.901283+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"402'4424","reported_seq":4440,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.498466+0000","last_change":"2024-09-13T15:05:11.537076+0000","last_active":"2024-09-13T15:06:01.498466+0000","last_peered":"2024-09-13T15:06:01.498466+0000","last_clean":"2024-09-13T15:06:01.498466+0000","last_became_active":"2024-09-13T15:05:11.536775+0000","last_became_peered":"2024-09-13T15:05:11.536775+0000","last_unstale":"2024-09-13T15:06:01.498466+0000","last_undegraded":"2024-09-13T15:06:01.498466+0000","last_fullsized":"2024-09-13T15:06:01.498466+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:38.213878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4424,"num_write_kb":283136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":481,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.896208+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:05:50.896208+0000","last_peered":"2024-09-13T15:05:50.896208+0000","last_clean":"2024-09-13T15:05:50.896208+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:05:50.896208+0000","last_undegraded":"2024-09-13T15:05:50.896208+0000","last_fullsized":"2024-09-13T15:05:50.896208+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":425,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.828042+0000","last_change":"2024-09-13T14:59:12.688845+0000","last_active":"2024-09-13T15:05:39.828042+0000","last_peered":"2024-09-13T15:05:39.828042+0000","last_clean":"2024-09-13T15:05:39.828042+0000","last_became_active":"2024-09-13T14:59:12.688415+0000","last_became_peered":"2024-09-13T14:59:12.688415+0000","last_unstale":"2024-09-13T15:05:39.828042+0000","last_undegraded":"2024-09-13T15:05:39.828042+0000","last_fullsized":"2024-09-13T15:05:39.828042+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:33:43.791192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":90,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.902299+0000","last_change":"2024-09-13T14:59:12.712649+0000","last_active":"2024-09-13T15:05:50.902299+0000","last_peered":"2024-09-13T15:05:50.902299+0000","last_clean":"2024-09-13T15:05:50.902299+0000","last_became_active":"2024-09-13T14:59:12.709542+0000","last_became_peered":"2024-09-13T14:59:12.709542+0000","last_unstale":"2024-09-13T15:05:50.902299+0000","last_undegraded":"2024-09-13T15:05:50.902299+0000","last_fullsized":"2024-09-13T15:05:50.902299+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:19:49.186316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"402'4317","reported_seq":4333,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.458709+0000","last_change":"2024-09-13T15:05:11.105322+0000","last_active":"2024-09-13T15:06:01.458709+0000","last_peered":"2024-09-13T15:06:01.458709+0000","last_clean":"2024-09-13T15:06:01.458709+0000","last_became_active":"2024-09-13T15:05:11.104691+0000","last_became_peered":"2024-09-13T15:05:11.104691+0000","last_unstale":"2024-09-13T15:06:01.458709+0000","last_undegraded":"2024-09-13T15:06:01.458709+0000","last_fullsized":"2024-09-13T15:06:01.458709+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4317,"log_dups_size":0,"ondisk_log_size":4317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:23.376418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282918912,"num_objects":4317,"num_object_clones":0,"num_object_copies":12951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4317,"num_write_kb":276288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":90,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:46.502132+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:05:46.502132+0000","last_peered":"2024-09-13T15:05:46.502132+0000","last_clean":"2024-09-13T15:05:46.502132+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:05:46.502132+0000","last_undegraded":"2024-09-13T15:05:46.502132+0000","last_fullsized":"2024-09-13T15:05:46.502132+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":90,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.900918+0000","last_change":"2024-09-13T14:59:12.709883+0000","last_active":"2024-09-13T15:05:50.900918+0000","last_peered":"2024-09-13T15:05:50.900918+0000","last_clean":"2024-09-13T15:05:50.900918+0000","last_became_active":"2024-09-13T14:59:12.709383+0000","last_became_peered":"2024-09-13T14:59:12.709383+0000","last_unstale":"2024-09-13T15:05:50.900918+0000","last_undegraded":"2024-09-13T15:05:50.900918+0000","last_fullsized":"2024-09-13T15:05:50.900918+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:14:44.247203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"402'4441","reported_seq":4457,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.484344+0000","last_change":"2024-09-13T15:05:11.103992+0000","last_active":"2024-09-13T15:06:01.484344+0000","last_peered":"2024-09-13T15:06:01.484344+0000","last_clean":"2024-09-13T15:06:01.484344+0000","last_became_active":"2024-09-13T15:05:11.103694+0000","last_became_peered":"2024-09-13T15:05:11.103694+0000","last_unstale":"2024-09-13T15:06:01.484344+0000","last_undegraded":"2024-09-13T15:06:01.484344+0000","last_fullsized":"2024-09-13T15:06:01.484344+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:15.405086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291045376,"num_objects":4441,"num_object_clones":0,"num_object_copies":13323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4441,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":481,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893205+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:05:50.893205+0000","last_peered":"2024-09-13T15:05:50.893205+0000","last_clean":"2024-09-13T15:05:50.893205+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:05:50.893205+0000","last_undegraded":"2024-09-13T15:05:50.893205+0000","last_fullsized":"2024-09-13T15:05:50.893205+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":634,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.899592+0000","last_change":"2024-09-13T14:40:37.181925+0000","last_active":"2024-09-13T15:05:50.899592+0000","last_peered":"2024-09-13T15:05:50.899592+0000","last_clean":"2024-09-13T15:05:50.899592+0000","last_became_active":"2024-09-13T14:40:37.181542+0000","last_became_peered":"2024-09-13T14:40:37.181542+0000","last_unstale":"2024-09-13T15:05:50.899592+0000","last_undegraded":"2024-09-13T15:05:50.899592+0000","last_fullsized":"2024-09-13T15:05:50.899592+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:35:18.558418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":647,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890733+0000","last_change":"2024-09-13T14:59:12.708447+0000","last_active":"2024-09-13T15:05:50.890733+0000","last_peered":"2024-09-13T15:05:50.890733+0000","last_clean":"2024-09-13T15:05:50.890733+0000","last_became_active":"2024-09-13T14:59:12.708107+0000","last_became_peered":"2024-09-13T14:59:12.708107+0000","last_unstale":"2024-09-13T15:05:50.890733+0000","last_undegraded":"2024-09-13T15:05:50.890733+0000","last_fullsized":"2024-09-13T15:05:50.890733+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:07:30.015603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"402'4540","reported_seq":4565,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.353411+0000","last_change":"2024-09-13T15:05:11.105485+0000","last_active":"2024-09-13T15:06:02.353411+0000","last_peered":"2024-09-13T15:06:02.353411+0000","last_clean":"2024-09-13T15:06:02.353411+0000","last_became_active":"2024-09-13T15:05:11.105170+0000","last_became_peered":"2024-09-13T15:05:11.105170+0000","last_unstale":"2024-09-13T15:06:02.353411+0000","last_undegraded":"2024-09-13T15:06:02.353411+0000","last_fullsized":"2024-09-13T15:06:02.353411+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4540,"log_dups_size":0,"ondisk_log_size":4540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:06:35.316067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297533440,"num_objects":4540,"num_object_clones":0,"num_object_copies":13620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4540,"num_write_kb":290560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":602,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:46.451274+0000","last_change":"2024-09-13T15:03:46.574157+0000","last_active":"2024-09-13T15:05:46.451274+0000","last_peered":"2024-09-13T15:05:46.451274+0000","last_clean":"2024-09-13T15:05:46.451274+0000","last_became_active":"2024-09-13T15:03:46.572750+0000","last_became_peered":"2024-09-13T15:03:46.572750+0000","last_unstale":"2024-09-13T15:05:46.451274+0000","last_undegraded":"2024-09-13T15:05:46.451274+0000","last_fullsized":"2024-09-13T15:05:46.451274+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:57:55.892848+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":119,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893677+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:05:50.893677+0000","last_peered":"2024-09-13T15:05:50.893677+0000","last_clean":"2024-09-13T15:05:50.893677+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:05:50.893677+0000","last_undegraded":"2024-09-13T15:05:50.893677+0000","last_fullsized":"2024-09-13T15:05:50.893677+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":620,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.900096+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:05:50.900096+0000","last_peered":"2024-09-13T15:05:50.900096+0000","last_clean":"2024-09-13T15:05:50.900096+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:05:50.900096+0000","last_undegraded":"2024-09-13T15:05:50.900096+0000","last_fullsized":"2024-09-13T15:05:50.900096+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":387,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.828259+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:05:39.828259+0000","last_peered":"2024-09-13T15:05:39.828259+0000","last_clean":"2024-09-13T15:05:39.828259+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:05:39.828259+0000","last_undegraded":"2024-09-13T15:05:39.828259+0000","last_fullsized":"2024-09-13T15:05:39.828259+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"402'4559","reported_seq":4575,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.497995+0000","last_change":"2024-09-13T15:05:11.104165+0000","last_active":"2024-09-13T15:06:01.497995+0000","last_peered":"2024-09-13T15:06:01.497995+0000","last_clean":"2024-09-13T15:06:01.497995+0000","last_became_active":"2024-09-13T15:05:11.103885+0000","last_became_peered":"2024-09-13T15:05:11.103885+0000","last_unstale":"2024-09-13T15:06:01.497995+0000","last_undegraded":"2024-09-13T15:06:01.497995+0000","last_fullsized":"2024-09-13T15:06:01.497995+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:32.640246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4559,"num_write_kb":291776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":915,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893272+0000","last_change":"2024-09-13T14:59:15.698251+0000","last_active":"2024-09-13T15:05:50.893272+0000","last_peered":"2024-09-13T15:05:50.893272+0000","last_clean":"2024-09-13T15:05:50.893272+0000","last_became_active":"2024-09-13T14:59:15.697762+0000","last_became_peered":"2024-09-13T14:59:15.697762+0000","last_unstale":"2024-09-13T15:05:50.893272+0000","last_undegraded":"2024-09-13T15:05:50.893272+0000","last_fullsized":"2024-09-13T15:05:50.893272+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:14:27.936972+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":639,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890011+0000","last_change":"2024-09-13T15:03:44.117533+0000","last_active":"2024-09-13T15:05:50.890011+0000","last_peered":"2024-09-13T15:05:50.890011+0000","last_clean":"2024-09-13T15:05:50.890011+0000","last_became_active":"2024-09-13T15:03:44.117216+0000","last_became_peered":"2024-09-13T15:03:44.117216+0000","last_unstale":"2024-09-13T15:05:50.890011+0000","last_undegraded":"2024-09-13T15:05:50.890011+0000","last_fullsized":"2024-09-13T15:05:50.890011+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:06:17.222690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":194,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.902741+0000","last_change":"2024-09-13T15:03:44.115491+0000","last_active":"2024-09-13T15:05:50.902741+0000","last_peered":"2024-09-13T15:05:50.902741+0000","last_clean":"2024-09-13T15:05:50.902741+0000","last_became_active":"2024-09-13T15:03:44.114978+0000","last_became_peered":"2024-09-13T15:03:44.114978+0000","last_unstale":"2024-09-13T15:05:50.902741+0000","last_undegraded":"2024-09-13T15:05:50.902741+0000","last_fullsized":"2024-09-13T15:05:50.902741+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:04.996891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":473,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890560+0000","last_change":"2024-09-13T15:03:44.117741+0000","last_active":"2024-09-13T15:05:50.890560+0000","last_peered":"2024-09-13T15:05:50.890560+0000","last_clean":"2024-09-13T15:05:50.890560+0000","last_became_active":"2024-09-13T15:03:44.117404+0000","last_became_peered":"2024-09-13T15:03:44.117404+0000","last_unstale":"2024-09-13T15:05:50.890560+0000","last_undegraded":"2024-09-13T15:05:50.890560+0000","last_fullsized":"2024-09-13T15:05:50.890560+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:43:02.169360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"402'4450","reported_seq":4466,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.466471+0000","last_change":"2024-09-13T15:05:11.537213+0000","last_active":"2024-09-13T15:06:01.466471+0000","last_peered":"2024-09-13T15:06:01.466471+0000","last_clean":"2024-09-13T15:06:01.466471+0000","last_became_active":"2024-09-13T15:05:11.536909+0000","last_became_peered":"2024-09-13T15:05:11.536909+0000","last_unstale":"2024-09-13T15:06:01.466471+0000","last_undegraded":"2024-09-13T15:06:01.466471+0000","last_fullsized":"2024-09-13T15:06:01.466471+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4450,"log_dups_size":0,"ondisk_log_size":4450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:13.844350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291635200,"num_objects":4450,"num_object_clones":0,"num_object_copies":13350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4450,"num_write_kb":284800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":650,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.900951+0000","last_change":"2024-09-13T15:03:46.571609+0000","last_active":"2024-09-13T15:05:50.900951+0000","last_peered":"2024-09-13T15:05:50.900951+0000","last_clean":"2024-09-13T15:05:50.900951+0000","last_became_active":"2024-09-13T15:03:46.571111+0000","last_became_peered":"2024-09-13T15:03:46.571111+0000","last_unstale":"2024-09-13T15:05:50.900951+0000","last_undegraded":"2024-09-13T15:05:50.900951+0000","last_fullsized":"2024-09-13T15:05:50.900951+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:26.923160+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":690,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893569+0000","last_change":"2024-09-13T15:03:44.113114+0000","last_active":"2024-09-13T15:05:50.893569+0000","last_peered":"2024-09-13T15:05:50.893569+0000","last_clean":"2024-09-13T15:05:50.893569+0000","last_became_active":"2024-09-13T15:03:44.112798+0000","last_became_peered":"2024-09-13T15:03:44.112798+0000","last_unstale":"2024-09-13T15:05:50.893569+0000","last_undegraded":"2024-09-13T15:05:50.893569+0000","last_fullsized":"2024-09-13T15:05:50.893569+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:52:39.690606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":633,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890306+0000","last_change":"2024-09-13T14:40:02.371704+0000","last_active":"2024-09-13T15:05:50.890306+0000","last_peered":"2024-09-13T15:05:50.890306+0000","last_clean":"2024-09-13T15:05:50.890306+0000","last_became_active":"2024-09-13T14:40:02.371361+0000","last_became_peered":"2024-09-13T14:40:02.371361+0000","last_unstale":"2024-09-13T15:05:50.890306+0000","last_undegraded":"2024-09-13T15:05:50.890306+0000","last_fullsized":"2024-09-13T15:05:50.890306+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:49:26.344997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":700,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890512+0000","last_change":"2024-09-13T15:03:44.120103+0000","last_active":"2024-09-13T15:05:50.890512+0000","last_peered":"2024-09-13T15:05:50.890512+0000","last_clean":"2024-09-13T15:05:50.890512+0000","last_became_active":"2024-09-13T15:03:44.119715+0000","last_became_peered":"2024-09-13T15:03:44.119715+0000","last_unstale":"2024-09-13T15:05:50.890512+0000","last_undegraded":"2024-09-13T15:05:50.890512+0000","last_fullsized":"2024-09-13T15:05:50.890512+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:57:16.484960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.e","version":"402'4302","reported_seq":4318,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.512209+0000","last_change":"2024-09-13T15:05:11.104779+0000","last_active":"2024-09-13T15:06:01.512209+0000","last_peered":"2024-09-13T15:06:01.512209+0000","last_clean":"2024-09-13T15:06:01.512209+0000","last_became_active":"2024-09-13T15:05:11.104444+0000","last_became_peered":"2024-09-13T15:05:11.104444+0000","last_unstale":"2024-09-13T15:06:01.512209+0000","last_undegraded":"2024-09-13T15:06:01.512209+0000","last_fullsized":"2024-09-13T15:06:01.512209+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4302,"log_dups_size":0,"ondisk_log_size":4302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:23.452855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281935872,"num_objects":4302,"num_object_clones":0,"num_object_copies":12906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4302,"num_write_kb":275328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":444,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893220+0000","last_change":"2024-09-13T15:03:47.227943+0000","last_active":"2024-09-13T15:05:50.893220+0000","last_peered":"2024-09-13T15:05:50.893220+0000","last_clean":"2024-09-13T15:05:50.893220+0000","last_became_active":"2024-09-13T15:03:46.585701+0000","last_became_peered":"2024-09-13T15:03:46.585701+0000","last_unstale":"2024-09-13T15:05:50.893220+0000","last_undegraded":"2024-09-13T15:05:50.893220+0000","last_fullsized":"2024-09-13T15:05:50.893220+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:03:47.226199+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:03:47.226199+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:12.460412+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":124,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890963+0000","last_change":"2024-09-13T15:01:33.852433+0000","last_active":"2024-09-13T15:05:50.890963+0000","last_peered":"2024-09-13T15:05:50.890963+0000","last_clean":"2024-09-13T15:05:50.890963+0000","last_became_active":"2024-09-13T15:01:33.852015+0000","last_became_peered":"2024-09-13T15:01:33.852015+0000","last_unstale":"2024-09-13T15:05:50.890963+0000","last_undegraded":"2024-09-13T15:05:50.890963+0000","last_fullsized":"2024-09-13T15:05:50.890963+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:52:03.246939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":110,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826651+0000","last_change":"2024-09-13T15:03:44.120307+0000","last_active":"2024-09-13T15:05:39.826651+0000","last_peered":"2024-09-13T15:05:39.826651+0000","last_clean":"2024-09-13T15:05:39.826651+0000","last_became_active":"2024-09-13T15:03:44.119949+0000","last_became_peered":"2024-09-13T15:03:44.119949+0000","last_unstale":"2024-09-13T15:05:39.826651+0000","last_undegraded":"2024-09-13T15:05:39.826651+0000","last_fullsized":"2024-09-13T15:05:39.826651+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:39:52.569133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":645,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890011+0000","last_change":"2024-09-13T14:40:37.182105+0000","last_active":"2024-09-13T15:05:50.890011+0000","last_peered":"2024-09-13T15:05:50.890011+0000","last_clean":"2024-09-13T15:05:50.890011+0000","last_became_active":"2024-09-13T14:40:37.181744+0000","last_became_peered":"2024-09-13T14:40:37.181744+0000","last_unstale":"2024-09-13T15:05:50.890011+0000","last_undegraded":"2024-09-13T15:05:50.890011+0000","last_fullsized":"2024-09-13T15:05:50.890011+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:11:54.565995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.8","version":"402'4436","reported_seq":4461,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.855452+0000","last_change":"2024-09-13T15:05:11.105474+0000","last_active":"2024-09-13T15:06:01.855452+0000","last_peered":"2024-09-13T15:06:01.855452+0000","last_clean":"2024-09-13T15:06:01.855452+0000","last_became_active":"2024-09-13T15:05:11.105182+0000","last_became_peered":"2024-09-13T15:05:11.105182+0000","last_unstale":"2024-09-13T15:06:01.855452+0000","last_undegraded":"2024-09-13T15:06:01.855452+0000","last_fullsized":"2024-09-13T15:06:01.855452+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4436,"log_dups_size":0,"ondisk_log_size":4436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:03:16.270466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290717696,"num_objects":4436,"num_object_clones":0,"num_object_copies":13308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4436,"num_write_kb":283904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":654,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827242+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:05:39.827242+0000","last_peered":"2024-09-13T15:05:39.827242+0000","last_clean":"2024-09-13T15:05:39.827242+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:05:39.827242+0000","last_undegraded":"2024-09-13T15:05:39.827242+0000","last_fullsized":"2024-09-13T15:05:39.827242+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":213,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.902054+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:05:50.902054+0000","last_peered":"2024-09-13T15:05:50.902054+0000","last_clean":"2024-09-13T15:05:50.902054+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:05:50.902054+0000","last_undegraded":"2024-09-13T15:05:50.902054+0000","last_fullsized":"2024-09-13T15:05:50.902054+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":715,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890223+0000","last_change":"2024-09-13T15:03:44.116244+0000","last_active":"2024-09-13T15:05:50.890223+0000","last_peered":"2024-09-13T15:05:50.890223+0000","last_clean":"2024-09-13T15:05:50.890223+0000","last_became_active":"2024-09-13T15:03:44.115926+0000","last_became_peered":"2024-09-13T15:03:44.115926+0000","last_unstale":"2024-09-13T15:05:50.890223+0000","last_undegraded":"2024-09-13T15:05:50.890223+0000","last_fullsized":"2024-09-13T15:05:50.890223+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:01.231884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":571,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890818+0000","last_change":"2024-09-13T14:59:12.691942+0000","last_active":"2024-09-13T15:05:50.890818+0000","last_peered":"2024-09-13T15:05:50.890818+0000","last_clean":"2024-09-13T15:05:50.890818+0000","last_became_active":"2024-09-13T14:59:12.691085+0000","last_became_peered":"2024-09-13T14:59:12.691085+0000","last_unstale":"2024-09-13T15:05:50.890818+0000","last_undegraded":"2024-09-13T15:05:50.890818+0000","last_fullsized":"2024-09-13T15:05:50.890818+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:26.825376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.9","version":"402'4550","reported_seq":4575,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.343996+0000","last_change":"2024-09-13T15:05:11.537327+0000","last_active":"2024-09-13T15:06:02.343996+0000","last_peered":"2024-09-13T15:06:02.343996+0000","last_clean":"2024-09-13T15:06:02.343996+0000","last_became_active":"2024-09-13T15:05:11.536882+0000","last_became_peered":"2024-09-13T15:05:11.536882+0000","last_unstale":"2024-09-13T15:06:02.343996+0000","last_undegraded":"2024-09-13T15:06:02.343996+0000","last_fullsized":"2024-09-13T15:06:02.343996+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4550,"log_dups_size":0,"ondisk_log_size":4550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:10:14.728744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298188800,"num_objects":4550,"num_object_clones":0,"num_object_copies":13650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4550,"num_write_kb":291200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":720,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827520+0000","last_change":"2024-09-13T15:03:46.574886+0000","last_active":"2024-09-13T15:05:39.827520+0000","last_peered":"2024-09-13T15:05:39.827520+0000","last_clean":"2024-09-13T15:05:39.827520+0000","last_became_active":"2024-09-13T15:03:46.574563+0000","last_became_peered":"2024-09-13T15:03:46.574563+0000","last_unstale":"2024-09-13T15:05:39.827520+0000","last_undegraded":"2024-09-13T15:05:39.827520+0000","last_fullsized":"2024-09-13T15:05:39.827520+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:29.856598+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":632,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890116+0000","last_change":"2024-09-13T14:40:02.370151+0000","last_active":"2024-09-13T15:05:50.890116+0000","last_peered":"2024-09-13T15:05:50.890116+0000","last_clean":"2024-09-13T15:05:50.890116+0000","last_became_active":"2024-09-13T14:40:02.369517+0000","last_became_peered":"2024-09-13T14:40:02.369517+0000","last_unstale":"2024-09-13T15:05:50.890116+0000","last_undegraded":"2024-09-13T15:05:50.890116+0000","last_fullsized":"2024-09-13T15:05:50.890116+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:07:54.820278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":194,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.902370+0000","last_change":"2024-09-13T15:03:44.121114+0000","last_active":"2024-09-13T15:05:50.902370+0000","last_peered":"2024-09-13T15:05:50.902370+0000","last_clean":"2024-09-13T15:05:50.902370+0000","last_became_active":"2024-09-13T15:03:44.120758+0000","last_became_peered":"2024-09-13T15:03:44.120758+0000","last_unstale":"2024-09-13T15:05:50.902370+0000","last_undegraded":"2024-09-13T15:05:50.902370+0000","last_fullsized":"2024-09-13T15:05:50.902370+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:54.923156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":71,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893097+0000","last_change":"2024-09-13T15:03:44.114482+0000","last_active":"2024-09-13T15:05:50.893097+0000","last_peered":"2024-09-13T15:05:50.893097+0000","last_clean":"2024-09-13T15:05:50.893097+0000","last_became_active":"2024-09-13T15:03:44.112887+0000","last_became_peered":"2024-09-13T15:03:44.112887+0000","last_unstale":"2024-09-13T15:05:50.893097+0000","last_undegraded":"2024-09-13T15:05:50.893097+0000","last_fullsized":"2024-09-13T15:05:50.893097+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:37.719076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.a","version":"402'4579","reported_seq":4604,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:01.845650+0000","last_change":"2024-09-13T15:05:11.537005+0000","last_active":"2024-09-13T15:06:01.845650+0000","last_peered":"2024-09-13T15:06:01.845650+0000","last_clean":"2024-09-13T15:06:01.845650+0000","last_became_active":"2024-09-13T15:05:11.536634+0000","last_became_peered":"2024-09-13T15:05:11.536634+0000","last_unstale":"2024-09-13T15:06:01.845650+0000","last_undegraded":"2024-09-13T15:06:01.845650+0000","last_fullsized":"2024-09-13T15:06:01.845650+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4579,"log_dups_size":0,"ondisk_log_size":4579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:42:37.713772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300089344,"num_objects":4579,"num_object_clones":0,"num_object_copies":13737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4579,"num_write_kb":293056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":583,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826357+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:05:39.826357+0000","last_peered":"2024-09-13T15:05:39.826357+0000","last_clean":"2024-09-13T15:05:39.826357+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:05:39.826357+0000","last_undegraded":"2024-09-13T15:05:39.826357+0000","last_fullsized":"2024-09-13T15:05:39.826357+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":664,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.898356+0000","last_change":"2024-09-13T15:03:44.131836+0000","last_active":"2024-09-13T15:05:50.898356+0000","last_peered":"2024-09-13T15:05:50.898356+0000","last_clean":"2024-09-13T15:05:50.898356+0000","last_became_active":"2024-09-13T15:03:44.120633+0000","last_became_peered":"2024-09-13T15:03:44.120633+0000","last_unstale":"2024-09-13T15:05:50.898356+0000","last_undegraded":"2024-09-13T15:05:50.898356+0000","last_fullsized":"2024-09-13T15:05:50.898356+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:52.065604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":646,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890730+0000","last_change":"2024-09-13T14:59:12.691674+0000","last_active":"2024-09-13T15:05:50.890730+0000","last_peered":"2024-09-13T15:05:50.890730+0000","last_clean":"2024-09-13T15:05:50.890730+0000","last_became_active":"2024-09-13T14:59:12.691346+0000","last_became_peered":"2024-09-13T14:59:12.691346+0000","last_unstale":"2024-09-13T15:05:50.890730+0000","last_undegraded":"2024-09-13T15:05:50.890730+0000","last_fullsized":"2024-09-13T15:05:50.890730+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:13:20.962520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":664,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.899443+0000","last_change":"2024-09-13T15:03:44.123687+0000","last_active":"2024-09-13T15:05:50.899443+0000","last_peered":"2024-09-13T15:05:50.899443+0000","last_clean":"2024-09-13T15:05:50.899443+0000","last_became_active":"2024-09-13T15:03:44.123135+0000","last_became_peered":"2024-09-13T15:03:44.123135+0000","last_unstale":"2024-09-13T15:05:50.899443+0000","last_undegraded":"2024-09-13T15:05:50.899443+0000","last_fullsized":"2024-09-13T15:05:50.899443+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:42.101656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"402'4492","reported_seq":4517,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:06:02.793508+0000","last_change":"2024-09-13T15:05:11.104176+0000","last_active":"2024-09-13T15:06:02.793508+0000","last_peered":"2024-09-13T15:06:02.793508+0000","last_clean":"2024-09-13T15:06:02.793508+0000","last_became_active":"2024-09-13T15:05:11.103879+0000","last_became_peered":"2024-09-13T15:05:11.103879+0000","last_unstale":"2024-09-13T15:06:02.793508+0000","last_undegraded":"2024-09-13T15:06:02.793508+0000","last_fullsized":"2024-09-13T15:06:02.793508+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:43:06.183789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294387712,"num_objects":4492,"num_object_clones":0,"num_object_copies":13476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4492,"num_write_kb":287488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":894,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.897813+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:05:50.897813+0000","last_peered":"2024-09-13T15:05:50.897813+0000","last_clean":"2024-09-13T15:05:50.897813+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:05:50.897813+0000","last_undegraded":"2024-09-13T15:05:50.897813+0000","last_fullsized":"2024-09-13T15:05:50.897813+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":426,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.891356+0000","last_change":"2024-09-13T15:03:44.127913+0000","last_active":"2024-09-13T15:05:50.891356+0000","last_peered":"2024-09-13T15:05:50.891356+0000","last_clean":"2024-09-13T15:05:50.891356+0000","last_became_active":"2024-09-13T15:03:44.121626+0000","last_became_peered":"2024-09-13T15:03:44.121626+0000","last_unstale":"2024-09-13T15:05:50.891356+0000","last_undegraded":"2024-09-13T15:05:50.891356+0000","last_fullsized":"2024-09-13T15:05:50.891356+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:43:24.441513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":391,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827376+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:05:39.827376+0000","last_peered":"2024-09-13T15:05:39.827376+0000","last_clean":"2024-09-13T15:05:39.827376+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:05:39.827376+0000","last_undegraded":"2024-09-13T15:05:39.827376+0000","last_fullsized":"2024-09-13T15:05:39.827376+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":90,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826037+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:05:39.826037+0000","last_peered":"2024-09-13T15:05:39.826037+0000","last_clean":"2024-09-13T15:05:39.826037+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:05:39.826037+0000","last_undegraded":"2024-09-13T15:05:39.826037+0000","last_fullsized":"2024-09-13T15:05:39.826037+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":89,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826684+0000","last_change":"2024-09-13T14:59:12.711103+0000","last_active":"2024-09-13T15:05:39.826684+0000","last_peered":"2024-09-13T15:05:39.826684+0000","last_clean":"2024-09-13T15:05:39.826684+0000","last_became_active":"2024-09-13T14:59:12.709892+0000","last_became_peered":"2024-09-13T14:59:12.709892+0000","last_unstale":"2024-09-13T15:05:39.826684+0000","last_undegraded":"2024-09-13T15:05:39.826684+0000","last_fullsized":"2024-09-13T15:05:39.826684+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:03:27.961050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":99,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826388+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:05:39.826388+0000","last_peered":"2024-09-13T15:05:39.826388+0000","last_clean":"2024-09-13T15:05:39.826388+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:05:39.826388+0000","last_undegraded":"2024-09-13T15:05:39.826388+0000","last_fullsized":"2024-09-13T15:05:39.826388+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":552,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890988+0000","last_change":"2024-09-13T14:41:21.823588+0000","last_active":"2024-09-13T15:05:50.890988+0000","last_peered":"2024-09-13T15:05:50.890988+0000","last_clean":"2024-09-13T15:05:50.890988+0000","last_became_active":"2024-09-13T14:41:21.823221+0000","last_became_peered":"2024-09-13T14:41:21.823221+0000","last_unstale":"2024-09-13T15:05:50.890988+0000","last_undegraded":"2024-09-13T15:05:50.890988+0000","last_fullsized":"2024-09-13T15:05:50.890988+0000","mapping_epoch":136,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:02:18.980426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":607,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893590+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:05:50.893590+0000","last_peered":"2024-09-13T15:05:50.893590+0000","last_clean":"2024-09-13T15:05:50.893590+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:05:50.893590+0000","last_undegraded":"2024-09-13T15:05:50.893590+0000","last_fullsized":"2024-09-13T15:05:50.893590+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":171,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.890900+0000","last_change":"2024-09-13T15:01:33.853899+0000","last_active":"2024-09-13T15:05:50.890900+0000","last_peered":"2024-09-13T15:05:50.890900+0000","last_clean":"2024-09-13T15:05:50.890900+0000","last_became_active":"2024-09-13T15:01:33.853515+0000","last_became_peered":"2024-09-13T15:01:33.853515+0000","last_unstale":"2024-09-13T15:05:50.890900+0000","last_undegraded":"2024-09-13T15:05:50.890900+0000","last_fullsized":"2024-09-13T15:05:50.890900+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:45:04.035329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":123,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826945+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:05:39.826945+0000","last_peered":"2024-09-13T15:05:39.826945+0000","last_clean":"2024-09-13T15:05:39.826945+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:05:39.826945+0000","last_undegraded":"2024-09-13T15:05:39.826945+0000","last_fullsized":"2024-09-13T15:05:39.826945+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":259,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.898203+0000","last_change":"2024-09-13T15:03:44.113646+0000","last_active":"2024-09-13T15:05:50.898203+0000","last_peered":"2024-09-13T15:05:50.898203+0000","last_clean":"2024-09-13T15:05:50.898203+0000","last_became_active":"2024-09-13T15:03:44.113342+0000","last_became_peered":"2024-09-13T15:03:44.113342+0000","last_unstale":"2024-09-13T15:05:50.898203+0000","last_undegraded":"2024-09-13T15:05:50.898203+0000","last_fullsized":"2024-09-13T15:05:50.898203+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:29.444600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":631,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.899362+0000","last_change":"2024-09-13T14:40:37.187314+0000","last_active":"2024-09-13T15:05:50.899362+0000","last_peered":"2024-09-13T15:05:50.899362+0000","last_clean":"2024-09-13T15:05:50.899362+0000","last_became_active":"2024-09-13T14:40:37.185848+0000","last_became_peered":"2024-09-13T14:40:37.185848+0000","last_unstale":"2024-09-13T15:05:50.899362+0000","last_undegraded":"2024-09-13T15:05:50.899362+0000","last_fullsized":"2024-09-13T15:05:50.899362+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:11.111169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":378,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.827315+0000","last_change":"2024-09-13T14:35:58.847523+0000","last_active":"2024-09-13T15:05:39.827315+0000","last_peered":"2024-09-13T15:05:39.827315+0000","last_clean":"2024-09-13T15:05:39.827315+0000","last_became_active":"2024-09-13T14:35:58.846010+0000","last_became_peered":"2024-09-13T14:35:58.846010+0000","last_unstale":"2024-09-13T15:05:39.827315+0000","last_undegraded":"2024-09-13T15:05:39.827315+0000","last_fullsized":"2024-09-13T15:05:39.827315+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:49:06.185904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":693,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.899516+0000","last_change":"2024-09-13T15:03:44.123370+0000","last_active":"2024-09-13T15:05:50.899516+0000","last_peered":"2024-09-13T15:05:50.899516+0000","last_clean":"2024-09-13T15:05:50.899516+0000","last_became_active":"2024-09-13T15:03:44.122679+0000","last_became_peered":"2024-09-13T15:03:44.122679+0000","last_unstale":"2024-09-13T15:05:50.899516+0000","last_undegraded":"2024-09-13T15:05:50.899516+0000","last_fullsized":"2024-09-13T15:05:50.899516+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:00:24.806181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":402,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901625+0000","last_change":"2024-09-13T15:03:44.127512+0000","last_active":"2024-09-13T15:05:50.901625+0000","last_peered":"2024-09-13T15:05:50.901625+0000","last_clean":"2024-09-13T15:05:50.901625+0000","last_became_active":"2024-09-13T15:03:44.126735+0000","last_became_peered":"2024-09-13T15:03:44.126735+0000","last_unstale":"2024-09-13T15:05:50.901625+0000","last_undegraded":"2024-09-13T15:05:50.901625+0000","last_fullsized":"2024-09-13T15:05:50.901625+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:11:20.140202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":644,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.892852+0000","last_change":"2024-09-13T14:59:12.691426+0000","last_active":"2024-09-13T15:05:50.892852+0000","last_peered":"2024-09-13T15:05:50.892852+0000","last_clean":"2024-09-13T15:05:50.892852+0000","last_became_active":"2024-09-13T14:59:12.691099+0000","last_became_peered":"2024-09-13T14:59:12.691099+0000","last_unstale":"2024-09-13T15:05:50.892852+0000","last_undegraded":"2024-09-13T15:05:50.892852+0000","last_fullsized":"2024-09-13T15:05:50.892852+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:18:56.953330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":645,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901586+0000","last_change":"2024-09-13T14:59:12.712272+0000","last_active":"2024-09-13T15:05:50.901586+0000","last_peered":"2024-09-13T15:05:50.901586+0000","last_clean":"2024-09-13T15:05:50.901586+0000","last_became_active":"2024-09-13T14:59:12.711880+0000","last_became_peered":"2024-09-13T14:59:12.711880+0000","last_unstale":"2024-09-13T15:05:50.901586+0000","last_undegraded":"2024-09-13T15:05:50.901586+0000","last_fullsized":"2024-09-13T15:05:50.901586+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:31:55.774404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":89,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.825914+0000","last_change":"2024-09-13T14:59:12.709280+0000","last_active":"2024-09-13T15:05:39.825914+0000","last_peered":"2024-09-13T15:05:39.825914+0000","last_clean":"2024-09-13T15:05:39.825914+0000","last_became_active":"2024-09-13T14:59:12.708170+0000","last_became_peered":"2024-09-13T14:59:12.708170+0000","last_unstale":"2024-09-13T15:05:39.825914+0000","last_undegraded":"2024-09-13T15:05:39.825914+0000","last_fullsized":"2024-09-13T15:05:39.825914+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:59.608331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":657,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893875+0000","last_change":"2024-09-13T14:40:37.180256+0000","last_active":"2024-09-13T15:05:50.893875+0000","last_peered":"2024-09-13T15:05:50.893875+0000","last_clean":"2024-09-13T15:05:50.893875+0000","last_became_active":"2024-09-13T14:40:37.179632+0000","last_became_peered":"2024-09-13T14:40:37.179632+0000","last_unstale":"2024-09-13T15:05:50.893875+0000","last_undegraded":"2024-09-13T15:05:50.893875+0000","last_fullsized":"2024-09-13T15:05:50.893875+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:45:27.115626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":75,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826603+0000","last_change":"2024-09-13T15:03:44.124264+0000","last_active":"2024-09-13T15:05:39.826603+0000","last_peered":"2024-09-13T15:05:39.826603+0000","last_clean":"2024-09-13T15:05:39.826603+0000","last_became_active":"2024-09-13T15:03:44.123914+0000","last_became_peered":"2024-09-13T15:03:44.123914+0000","last_unstale":"2024-09-13T15:05:39.826603+0000","last_undegraded":"2024-09-13T15:05:39.826603+0000","last_fullsized":"2024-09-13T15:05:39.826603+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:18:55.477300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":98,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826503+0000","last_change":"2024-09-13T15:04:24.852045+0000","last_active":"2024-09-13T15:05:39.826503+0000","last_peered":"2024-09-13T15:05:39.826503+0000","last_clean":"2024-09-13T15:05:39.826503+0000","last_became_active":"2024-09-13T15:04:24.851772+0000","last_became_peered":"2024-09-13T15:04:24.851772+0000","last_unstale":"2024-09-13T15:05:39.826503+0000","last_undegraded":"2024-09-13T15:05:39.826503+0000","last_fullsized":"2024-09-13T15:05:39.826503+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:53:57.699649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":371,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826300+0000","last_change":"2024-09-13T15:01:33.852507+0000","last_active":"2024-09-13T15:05:39.826300+0000","last_peered":"2024-09-13T15:05:39.826300+0000","last_clean":"2024-09-13T15:05:39.826300+0000","last_became_active":"2024-09-13T15:01:33.852196+0000","last_became_peered":"2024-09-13T15:01:33.852196+0000","last_unstale":"2024-09-13T15:05:39.826300+0000","last_undegraded":"2024-09-13T15:05:39.826300+0000","last_fullsized":"2024-09-13T15:05:39.826300+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:56:28.217977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.825824+0000","last_change":"2024-09-13T15:03:44.116627+0000","last_active":"2024-09-13T15:05:39.825824+0000","last_peered":"2024-09-13T15:05:39.825824+0000","last_clean":"2024-09-13T15:05:39.825824+0000","last_became_active":"2024-09-13T15:03:44.116197+0000","last_became_peered":"2024-09-13T15:03:44.116197+0000","last_unstale":"2024-09-13T15:05:39.825824+0000","last_undegraded":"2024-09-13T15:05:39.825824+0000","last_fullsized":"2024-09-13T15:05:39.825824+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:30.519779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":356,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.826296+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:05:39.826296+0000","last_peered":"2024-09-13T15:05:39.826296+0000","last_clean":"2024-09-13T15:05:39.826296+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:05:39.826296+0000","last_undegraded":"2024-09-13T15:05:39.826296+0000","last_fullsized":"2024-09-13T15:05:39.826296+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":695,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.898344+0000","last_change":"2024-09-13T15:03:44.120794+0000","last_active":"2024-09-13T15:05:50.898344+0000","last_peered":"2024-09-13T15:05:50.898344+0000","last_clean":"2024-09-13T15:05:50.898344+0000","last_became_active":"2024-09-13T15:03:44.120431+0000","last_became_peered":"2024-09-13T15:03:44.120431+0000","last_unstale":"2024-09-13T15:05:50.898344+0000","last_undegraded":"2024-09-13T15:05:50.898344+0000","last_fullsized":"2024-09-13T15:05:50.898344+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:27.396753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":654,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.901799+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:05:50.901799+0000","last_peered":"2024-09-13T15:05:50.901799+0000","last_clean":"2024-09-13T15:05:50.901799+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:05:50.901799+0000","last_undegraded":"2024-09-13T15:05:50.901799+0000","last_fullsized":"2024-09-13T15:05:50.901799+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":89,"reported_epoch":401,"state":"active+clean","last_fresh":"2024-09-13T15:05:39.825649+0000","last_change":"2024-09-13T14:59:12.709112+0000","last_active":"2024-09-13T15:05:39.825649+0000","last_peered":"2024-09-13T15:05:39.825649+0000","last_clean":"2024-09-13T15:05:39.825649+0000","last_became_active":"2024-09-13T14:59:12.708807+0000","last_became_peered":"2024-09-13T14:59:12.708807+0000","last_unstale":"2024-09-13T15:05:39.825649+0000","last_undegraded":"2024-09-13T15:05:39.825649+0000","last_fullsized":"2024-09-13T15:05:39.825649+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:19:18.152336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":90,"reported_epoch":402,"state":"active+clean","last_fresh":"2024-09-13T15:05:50.893887+0000","last_change":"2024-09-13T15:03:44.120052+0000","last_active":"2024-09-13T15:05:50.893887+0000","last_peered":"2024-09-13T15:05:50.893887+0000","last_clean":"2024-09-13T15:05:50.893887+0000","last_became_active":"2024-09-13T15:03:44.109611+0000","last_became_peered":"2024-09-13T15:03:44.109611+0000","last_unstale":"2024-09-13T15:05:50.893887+0000","last_undegraded":"2024-09-13T15:05:50.893887+0000","last_fullsized":"2024-09-13T15:05:50.893887+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:04:55.196794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":61,"num_bytes_recovered":5786,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":4713807872,"num_objects":71927,"num_object_clones":0,"num_object_copies":215781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":71927,"num_write_kb":4603328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14109835264,"data_stored":14109835264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71927,"ondisk_log_size":71927,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234551,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2466964,"kb_used_data":1120140,"kb_used_omap":1,"kb_used_meta":1346814,"kb_avail":91278188,"statfs":{"total":95995035648,"available":93468864512,"internally_reserved":0,"allocated":1147023360,"data_stored":1144849010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1379137995},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496189,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3283604,"kb_used_data":1978700,"kb_used_omap":1,"kb_used_meta":1304894,"kb_avail":90461548,"statfs":{"total":95995035648,"available":92632625152,"internally_reserved":0,"allocated":2026188800,"data_stored":2024014995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1336211911},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757827,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2824648,"kb_used_data":1409536,"kb_used_omap":1,"kb_used_meta":1415102,"kb_avail":90920504,"statfs":{"total":95995035648,"available":93102596096,"internally_reserved":0,"allocated":1443364864,"data_stored":1441198804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1449064902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019463,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3218132,"kb_used_data":1969612,"kb_used_omap":1,"kb_used_meta":1248510,"kb_avail":90527020,"statfs":{"total":95995035648,"available":92699668480,"internally_reserved":0,"allocated":2016882688,"data_stored":2014708358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1278474694},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650675,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3837572,"kb_used_data":2877372,"kb_used_omap":4,"kb_used_meta":960187,"kb_avail":89907580,"statfs":{"total":95995035648,"available":92065361920,"internally_reserved":0,"allocated":2946428928,"data_stored":2944266726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":983232047},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879610,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3004428,"kb_used_data":2028164,"kb_used_omap":6,"kb_used_meta":976249,"kb_avail":90740724,"statfs":{"total":95995035648,"available":92918501376,"internally_reserved":0,"allocated":2076839936,"data_stored":2074670158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":999679545},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,3,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435627,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3487740,"kb_used_data":2294260,"kb_used_omap":5,"kb_used_meta":1193466,"kb_avail":90257412,"statfs":{"total":95995035648,"available":92423589888,"internally_reserved":0,"allocated":2349322240,"data_stored":2347167091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1222109537},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":1494648619070,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2371616768,"data_stored":2371616768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2090336256,"data_stored":2090336256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2982805504,"data_stored":2982805504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2041774080,"data_stored":2041774080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1451819008,"data_stored":1451819008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2021261312,"data_stored":2021261312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1150222336,"data_stored":1150222336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:06:07.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_9 pgp_num 16 2024-09-13T15:06:07.846 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:07.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:07.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:07.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:08.003 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:08.013 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:07 smithi042 ceph-mon[108570]: pgmap v1453: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:06:08.013 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:07 smithi042 ceph-mon[113926]: pgmap v1453: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55042 13.3 13:c5766ce4:::benchmark_data_smithi042_286876_object55041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:06:08.015 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 278.568968627, 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25510091000000001, 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55042 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.016 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439902+0000", 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1839999999999999e-06 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.017 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439931+0000", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9566000000000001e-05 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439951+0000", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9743999999999999e-05 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439961+0000", 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0671e-05 2024-09-13T15:06:08.018 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440069+0000", 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107279 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440197+0000", 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012778299999999999 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.019 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440387+0000", 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019016000000000001 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.442194+0000", 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018073729999999999 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.445468+0000", 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032742719999999999 2024-09-13T15:06:08.020 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694893+0000", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.24942428 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694950+0000", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6963000000000003e-05 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694999+0000", 2024-09-13T15:06:08.021 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9635000000000002e-05 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55050 13.3 13:c347d548:::benchmark_data_smithi042_286876_object55049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 278.56262850100001, 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25196794900000002, 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.022 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55050 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.023 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446240+0000", 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0250000000000001e-06 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446264+0000", 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3869000000000001e-05 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.024 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446279+0000", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4280999999999999e-05 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446286+0000", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4780000000000002e-06 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446330+0000", 2024-09-13T15:06:08.025 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3436000000000002e-05 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446454+0000", 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012442200000000001 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446616+0000", 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000161881 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.026 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.448563+0000", 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019471880000000001 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.450965+0000", 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024016760000000002 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698076+0000", 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.247111365 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.027 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698159+0000", 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3280000000000002e-05 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698206+0000", 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7048000000000001e-05 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.028 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55064 13.3 13:c67e5ab2:::benchmark_data_smithi042_286876_object55063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 278.55271880800001, 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.24222567, 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55064 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.029 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.030 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456150+0000", 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6509999999999999e-06 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456171+0000", 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1084999999999999e-05 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456186+0000", 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4651000000000001e-05 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.031 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456193+0000", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2629999999999997e-06 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456252+0000", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8833999999999999e-05 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456419+0000", 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016782 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.032 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456566+0000", 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014662900000000001 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.458701+0000", 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021344670000000001 2024-09-13T15:06:08.033 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.459472+0000", 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00077120699999999999 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698268+0000", 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23879609299999999 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.034 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698329+0000", 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1620999999999998e-05 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698374+0000", 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4348999999999998e-05 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55075 13.3 13:ce764c83:::benchmark_data_smithi042_286876_object55074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:06:08.035 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 278.54504549400002, 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23468729499999999, 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55075 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.036 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463823+0000", 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4160000000000001e-06 2024-09-13T15:06:08.037 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463841+0000", 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8533999999999999e-05 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463854+0000", 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2129e-05 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.038 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463882+0000", 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8167999999999999e-05 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463938+0000", 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6517000000000001e-05 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.464028+0000", 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0132999999999999e-05 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.039 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.464119+0000", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0520999999999996e-05 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.465694+0000", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015750359999999999 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.466445+0000", 2024-09-13T15:06:08.040 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00075109500000000002 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698425+0000", 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23198013200000001 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698464+0000", 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9019e-05 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.041 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698509+0000", 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4595000000000002e-05 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60151 13.9 13:924bdd2a:::benchmark_data_smithi042_286876_object60150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.152764112, 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99705287799999998, 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.042 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60151 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.043 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.044 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.044 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:06:08.044 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856104+0000", 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.468e-06 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856132+0000", 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7880000000000001e-05 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.045 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856144+0000", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2063e-05 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856152+0000", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9710000000000004e-06 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863798+0000", 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076454499999999998 2024-09-13T15:06:08.046 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863807+0000", 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6600000000000007e-06 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.852422+0000", 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98861485800000004 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.047 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853156+0000", 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000733528 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60163 13.9 13:90dccfe2:::benchmark_data_smithi042_286876_object60162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.152697214, 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0136024189999999, 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.048 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60163 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.049 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0299999999999999e-07 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856193+0000", 2024-09-13T15:06:08.050 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3039000000000001e-05 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856202+0000", 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4919999999999993e-06 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856209+0000", 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.063e-06 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.051 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863839+0000", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076298069999999997 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863846+0000", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9500000000000004e-06 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853617+0000", 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98977083600000004 2024-09-13T15:06:08.052 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853839+0000", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000222084 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853992+0000", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015310000000000001 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862679+0000", 2024-09-13T15:06:08.053 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086871080000000007 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867225+0000", 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045461590000000001 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869604+0000", 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023790230000000001 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.054 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869720+0000", 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116223 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869772+0000", 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1931999999999997e-05 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.055 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60181 13.9 13:95295340:::benchmark_data_smithi042_286876_object60180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.13731809900003, 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99836792200000002, 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60181 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.056 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871550+0000", 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4099999999999998e-07 2024-09-13T15:06:08.057 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871574+0000", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4311000000000002e-05 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871585+0000", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1369000000000001e-05 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871593+0000", 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3509999999999996e-06 2024-09-13T15:06:08.058 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871655+0000", 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2860999999999993e-05 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871666+0000", 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0478e-05 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.059 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.855776+0000", 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98411022500000001 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.855909+0000", 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013322299999999999 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.856068+0000", 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015866099999999999 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.060 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867208+0000", 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011140097 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867348+0000", 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014019600000000001 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.061 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869817+0000", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024688209999999999 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869887+0000", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9543999999999997e-05 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869917+0000", 2024-09-13T15:06:08.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0043999999999999e-05 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60182 13.9 13:9ddd2598:::benchmark_data_smithi042_286876_object60181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.13723896699997, 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99860969899999996, 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.063 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60182 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871629+0000", 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7999999999999995e-07 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871660+0000", 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0911999999999998e-05 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871669+0000", 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6020000000000006e-06 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871676+0000", 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6579999999999996e-06 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871872+0000", 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000195797 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871881+0000", 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2900000000000008e-06 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.857719+0000", 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98583839399999995 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.857873+0000", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015369400000000001 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.858020+0000", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014712699999999999 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867265+0000", 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0092451479999999999 2024-09-13T15:06:08.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867355+0000", 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9481000000000006e-05 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870168+0000", 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002813505 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870198+0000", 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9811999999999998e-05 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870238+0000", 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9598999999999998e-05 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.070 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60194 13.9 13:9e098fbd:::benchmark_data_smithi042_286876_object60193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.12930227599998, 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99081947299999995, 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.071 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60194 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879566+0000", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3540000000000001e-06 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879587+0000", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0975000000000002e-05 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879602+0000", 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4613999999999999e-05 2024-09-13T15:06:08.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879609+0000", 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3579999999999997e-06 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879655+0000", 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6029000000000003e-05 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879661+0000", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7790000000000001e-06 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860158+0000", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98049688099999999 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860273+0000", 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000115148 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860394+0000", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012133000000000001 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867336+0000", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069417280000000003 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867364+0000", 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7767999999999999e-05 2024-09-13T15:06:08.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870310+0000", 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029460110000000001 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870341+0000", 2024-09-13T15:06:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1698e-05 2024-09-13T15:06:08.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870384+0000", 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2799999999999997e-05 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60196 13.9 13:9c14fad6:::benchmark_data_smithi042_286876_object60195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.12622074500001, 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98790771499999996, 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.081 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60196 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882648+0000", 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.525e-06 2024-09-13T15:06:08.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882672+0000", 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4334000000000001e-05 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882687+0000", 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5045000000000001e-05 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882696+0000", 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.405e-06 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882770+0000", 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4103000000000005e-05 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882782+0000", 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2369e-05 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862710+0000", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979927667 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862823+0000", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113725 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862977+0000", 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153203 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867369+0000", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004392796 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867430+0000", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0168999999999999e-05 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870441+0000", 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030115739999999999 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870501+0000", 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9806000000000003e-05 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870554+0000", 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2994000000000001e-05 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.088 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60200 13.9 13:99cafdac:::benchmark_data_smithi042_286876_object60199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.12371325200002, 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98952164600000003, 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60200 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885156+0000", 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3420000000000002e-06 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885179+0000", 2024-09-13T15:06:08.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3436999999999999e-05 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885201+0000", 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1253000000000001e-05 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885212+0000", 2024-09-13T15:06:08.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0804e-05 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885289+0000", 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7435000000000007e-05 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885300+0000", 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0947999999999999e-05 2024-09-13T15:06:08.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865182+0000", 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979881954 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865299+0000", 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011673200000000001 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865433+0000", 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013482699999999999 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869562+0000", 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041289170000000002 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870619+0000", 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010570009999999999 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874577+0000", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003957899 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874632+0000", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5188000000000001e-05 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874675+0000", 2024-09-13T15:06:08.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2908999999999999e-05 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60202 13.9 13:9b86202e:::benchmark_data_smithi042_286876_object60201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 275.122539975, 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98848820400000004, 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.097 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60202 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886328+0000", 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.376e-06 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886352+0000", 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3530999999999999e-05 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886368+0000", 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6064000000000001e-05 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886377+0000", 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6859999999999995e-06 2024-09-13T15:06:08.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886442+0000", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4979000000000002e-05 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886453+0000", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1392999999999999e-05 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867503+0000", 2024-09-13T15:06:08.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98104998499999996 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867622+0000", 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000118539 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867722+0000", 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100704 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870249+0000", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025271690000000001 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870906+0000", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000656523 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874725+0000", 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038195 2024-09-13T15:06:08.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874771+0000", 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6014999999999999e-05 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874815+0000", 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3739999999999998e-05 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.104 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89942 13.3 13:cd1c33de:::benchmark_data_smithi042_286876_object89941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 249.69396256100001, 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20837172000000001, 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89942 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314906+0000", 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2920000000000001e-06 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314928+0000", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.194e-05 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314939+0000", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1776e-05 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314946+0000", 2024-09-13T15:06:08.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0330000000000003e-06 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.319828+0000", 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048817699999999997 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.319962+0000", 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000134227 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320106+0000", 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000144067 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323426+0000", 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033193440000000001 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326450+0000", 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003023759 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523109+0000", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.196659522 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523220+0000", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011048 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523276+0000", 2024-09-13T15:06:08.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6509999999999999e-05 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89954 13.3 13:c3093c51:::benchmark_data_smithi042_286876_object89953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 249.682448112, 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.21377985499999999, 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.111 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89954 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326421+0000", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8360000000000001e-06 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326444+0000", 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3133999999999999e-05 2024-09-13T15:06:08.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326456+0000", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2500999999999999e-05 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326464+0000", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2370000000000003e-06 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326791+0000", 2024-09-13T15:06:08.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000327917 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327063+0000", 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00027153300000000002 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327469+0000", 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00040561699999999999 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334420+0000", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069510140000000002 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334463+0000", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3658000000000001e-05 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.539968+0000", 2024-09-13T15:06:08.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20550506900000001 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.540154+0000", 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000185768 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.540199+0000", 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4570999999999997e-05 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89957 13.3 13:c7f138f4:::benchmark_data_smithi042_286876_object89956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 249.68211259200001, 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.21694497500000001, 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89957 2024-09-13T15:06:08.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.119 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326755+0000", 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0270000000000001e-06 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326778+0000", 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2809000000000001e-05 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326791+0000", 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2849e-05 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326799+0000", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3890000000000005e-06 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.330353+0000", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035536000000000001 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.330768+0000", 2024-09-13T15:06:08.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041488099999999997 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.331300+0000", 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00053230699999999998 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334495+0000", 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003195008 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.339789+0000", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052936379999999998 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543579+0000", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20378991099999999 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543646+0000", 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7162999999999994e-05 2024-09-13T15:06:08.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543699+0000", 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3393000000000002e-05 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89967 13.3 13:c148241f:::benchmark_data_smithi042_286876_object89966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:06:08.124 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 249.67165780100001, 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20677611500000001, 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89967 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337211+0000", 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.703e-06 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337232+0000", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1271999999999999e-05 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337248+0000", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5954e-05 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337255+0000", 2024-09-13T15:06:08.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3710000000000002e-06 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337315+0000", 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9293000000000003e-05 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337416+0000", 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010092800000000001 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337546+0000", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012996199999999999 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.339985+0000", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024393319999999998 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.340731+0000", 2024-09-13T15:06:08.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00074605299999999999 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543883+0000", 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20315203800000001 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543946+0000", 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3051000000000003e-05 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543985+0000", 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9158e-05 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89985 13.3 13:ce68edab:::benchmark_data_smithi042_286876_object89984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 249.64669660300001, 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19764711200000001, 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.131 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89985 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362172+0000", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4959999999999999e-06 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362194+0000", 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2582000000000002e-05 2024-09-13T15:06:08.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362210+0000", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5968999999999999e-05 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362218+0000", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1910000000000003e-06 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362273+0000", 2024-09-13T15:06:08.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5087999999999998e-05 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362369+0000", 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6502999999999994e-05 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.135 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:08.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362462+0000", 2024-09-13T15:06:08.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3317000000000001e-05 2024-09-13T15:06:08.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.364568+0000", 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021051519999999999 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.365147+0000", 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00057914499999999999 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559557+0000", 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19441054299999999 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559733+0000", 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017600499999999999 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559817+0000", 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4121000000000003e-05 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:165999 13.3 13:c1fbbb06:::benchmark_data_smithi042_286876_object42498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 168.90291084500001, 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19437074400000001, 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 165999 2024-09-13T15:06:08.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105958+0000", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5770000000000001e-06 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105964+0000", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0020000000000003e-06 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105975+0000", 2024-09-13T15:06:08.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1447999999999999e-05 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105981+0000", 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1090000000000003e-06 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106035+0000", 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4153999999999997e-05 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106180+0000", 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014459400000000001 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106301+0000", 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120884 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300202+0000", 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19390109999999999 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300245+0000", 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2738999999999999e-05 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300253+0000", 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9789999999999993e-06 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300284+0000", 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1139999999999997e-05 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300327+0000", 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3018e-05 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.145 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:166001 13.3 13:c2d18c6b:::benchmark_data_smithi042_286876_object42500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 168.902465471, 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19413623299999999, 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 166001 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.146 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106403+0000", 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.15e-06 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106407+0000", 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7469999999999996e-06 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106419+0000", 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2013e-05 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106427+0000", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4410000000000004e-06 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110401+0000", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039738819999999998 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110506+0000", 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010556999999999999 2024-09-13T15:06:08.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110613+0000", 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010705300000000001 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300337+0000", 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.189723578 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300384+0000", 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6999999999999997e-05 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300429+0000", 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5194999999999997e-05 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300486+0000", 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6708e-05 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300538+0000", 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1895999999999999e-05 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:166006 13.3 13:cae3698d:::benchmark_data_smithi042_286876_object42505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 168.89993374400001, 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20369686100000001, 2024-09-13T15:06:08.152 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 166006 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108935+0000", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-06 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108939+0000", 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8559999999999998e-06 2024-09-13T15:06:08.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108950+0000", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0804e-05 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108958+0000", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7360000000000001e-06 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115116+0000", 2024-09-13T15:06:08.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061578730000000003 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115226+0000", 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011048999999999999 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115338+0000", 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011165100000000001 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300614+0000", 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18527559499999999 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300628+0000", 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4177e-05 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312470+0000", 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011842455999999999 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312581+0000", 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011113800000000001 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312630+0000", 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8773999999999998e-05 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:06:08.158 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:06:08.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:08.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:07 smithi159 ceph-mon[102519]: pgmap v1453: 99 pgs: 99 active+clean; 4.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-13T15:06:08.300 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:08.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:08.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:08.445 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:08.468 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 44 16 79667 79651 113.121 73.9375 0.0129229 0.00883488 2024-09-13T15:06:08.472 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:08.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:08.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:08.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:08.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:08.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.743 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:08.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:08.845 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:08.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:08.880 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:08.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:08.917 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/306150146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:08.917 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/306150146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:08.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:09.009 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:09.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:09.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.217 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/306150146' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:09.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:09.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:09.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 45 16 80913 80897 112.336 77.875 0.0096902 0.00889655 2024-09-13T15:06:09.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:09.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:09.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:06:09.638 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:09.741 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:09.785 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:09.795 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:09.796 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:09.797 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:09.797 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:09.797 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:09.797 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:09.797 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:09.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:09 smithi042 ceph-mon[113926]: pgmap v1454: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:06:09.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:09 smithi042 ceph-mon[108570]: pgmap v1454: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:06:09.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:09] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-13T15:06:09.887 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:09.935 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:09.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:09.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:10.037 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:10.067 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:10.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:10.090 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:10.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:10.215 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:09 smithi159 ceph-mon[102519]: pgmap v1454: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-13T15:06:10.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:10.353 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:10.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:10.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 46 16 82124 82108 111.539 75.6875 0.0138176 0.00896034 2024-09-13T15:06:10.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:10.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:10.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:10.632 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:10.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:10.764 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:10.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:10.866 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:10.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:10.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:10.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:10.940 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:11.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.057 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:11.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:11.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:11.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:11.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.351 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:11.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:11.452 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:11.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 47 16 83387 83371 110.845 78.9375 0.0266858 0.0090161 2024-09-13T15:06:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:11.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:11.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:11.625 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:11.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:11.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.773 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:11.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:11.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:11.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:11.914 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:11.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:12.015 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:12.045 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:12.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:12.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:11 smithi042 ceph-mon[113926]: pgmap v1455: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-13T15:06:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:11 smithi042 ceph-mon[108570]: pgmap v1455: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-13T15:06:12.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.179 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:12.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:11 smithi159 ceph-mon[102519]: pgmap v1455: 99 pgs: 99 active+clean; 4.8 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-13T15:06:12.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:12.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:12.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:12.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:12.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.447 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:12.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:12.469 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 48 15 84658 84643 110.192 79.5 0.00543576 0.00906988 2024-09-13T15:06:12.548 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:12.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:12.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:12.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:12.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:12.847 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:12.881 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:12.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:12.974 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:06:12.981 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:13.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.182 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:13.206 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:13.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.334 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:13.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:13.435 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:13.468 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:13.470 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 49 16 85766 85750 109.355 69.1875 0.0181993 0.0091393 2024-09-13T15:06:13.570 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:13.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:13.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:13.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:13.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:13.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:13.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:13.884 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:13.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:13.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:14.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:14.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:13 smithi042 ceph-mon[113926]: pgmap v1456: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T15:06:14.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:13 smithi042 ceph-mon[108570]: pgmap v1456: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T15:06:14.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:14.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:13 smithi159 ceph-mon[102519]: pgmap v1456: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-13T15:06:14.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:14.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:14.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.440 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:14.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:14.470 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 50 16 86969 86953 108.67 75.1875 0.0184185 0.00919475 2024-09-13T15:06:14.482 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:14.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:14.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:14.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:14.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:14.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:14.794 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:14.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:14.841 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:06:14.919 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:14.959 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:14.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:14.973 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:14.973 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:15.061 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:15.098 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:15.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:15.200 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:15.240 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:15.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:06:15.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:15.370 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:15.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:15.473 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 51 16 88012 87996 107.817 65.1875 0.00498438 0.00926871 2024-09-13T15:06:15.504 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:15.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:15.634 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:15.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:15.736 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:15.767 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:15.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:15.869 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:15.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:15.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:15.995 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:16.027 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[113926]: pgmap v1457: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:06:16.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[113926]: osdmap e403: 8 total, 8 up, 7 in 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[108570]: pgmap v1457: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:15 smithi042 ceph-mon[108570]: osdmap e403: 8 total, 8 up, 7 in 2024-09-13T15:06:16.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.167 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:16.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:15 smithi159 ceph-mon[102519]: pgmap v1457: 99 pgs: 99 active+clean; 5.1 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-13T15:06:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:15 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/741184984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:06:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:15 smithi159 ceph-mon[102519]: osdmap e403: 8 total, 8 up, 7 in 2024-09-13T15:06:16.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:16.306 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:16.408 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:16.438 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:16.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:16.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 52 16 89232 89216 107.21 76.25 0.0130505 0.00932283 2024-09-13T15:06:16.539 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:16.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:16.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:16.641 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:16.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:16.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:16.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:16.837 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:16.879 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:16.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:16.980 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:17.120 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.153 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.254 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:17.288 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:17.390 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.425 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:17.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 53 16 90353 90337 106.509 70.0625 0.0289605 0.00938315 2024-09-13T15:06:17.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:17.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:17.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:17.791 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:17.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:17.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:17.852 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 14 pgp_num to 16 2024-09-13T15:06:17.940 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:17.979 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:17.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:18.081 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[113926]: pgmap v1459: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T15:06:18.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/177806887' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[108570]: pgmap v1459: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T15:06:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/177806887' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:17 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.112 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:18.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:18.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:17 smithi159 ceph-mon[102519]: pgmap v1459: 99 pgs: 99 active+clean; 5.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 75 MiB/s wr, 1.21k op/s 2024-09-13T15:06:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:17 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/177806887' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:17 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"} : dispatch 2024-09-13T15:06:18.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:18.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:18.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:18.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:06:18.444 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:18.444 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:18.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 54 16 91532 91516 105.901 73.6875 0.0157522 0.00943707 2024-09-13T15:06:18.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:18.537 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:18.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:18.641 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:18.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.686 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:18.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:18.787 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.823 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:18.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:18.925 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:18.957 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:18.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:19.059 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:19.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:18 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:19.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:18 smithi042 ceph-mon[113926]: osdmap e404: 8 total, 8 up, 7 in 2024-09-13T15:06:19.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:18 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:19.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:18 smithi042 ceph-mon[108570]: osdmap e404: 8 total, 8 up, 7 in 2024-09-13T15:06:19.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:19.235 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:18 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num", "val": "16"}]': finished 2024-09-13T15:06:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:18 smithi159 ceph-mon[102519]: osdmap e404: 8 total, 8 up, 7 in 2024-09-13T15:06:19.337 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:19.372 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:19.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 55 15 92638 92623 105.234 69.1875 0.00817305 0.00949737 2024-09-13T15:06:19.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:19.611 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:19.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:19.741 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:19.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:19.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:19.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:19.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:19.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:20.008 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:20.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:20.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:19 smithi042 ceph-mon[108570]: pgmap v1461: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:06:20.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:19 smithi042 ceph-mon[113926]: pgmap v1461: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:06:20.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:19] "GET /metrics HTTP/1.1" 200 36530 "" "Prometheus/2.51.0" 2024-09-13T15:06:20.150 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:20.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:19 smithi159 ceph-mon[102519]: pgmap v1461: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-13T15:06:20.190 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:20.292 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:20.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:20.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:20.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 56 16 93770 93754 104.617 70.6875 0.010983 0.00955392 2024-09-13T15:06:20.473 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:20.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:20.750 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:20.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:20.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:20.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:21.005 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:21.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:21.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:21.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:21.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:06:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:21.418 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:21.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 57 16 94804 94788 103.915 64.625 0.0103698 0.00961702 2024-09-13T15:06:21.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:21.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.619 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:21.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.720 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.760 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:21.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:21.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:21.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:21.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:21.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:22.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:22.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:22.100 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:21 smithi159 ceph-mon[102519]: pgmap v1462: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:06:22.138 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:22.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.166 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:22.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:22.267 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:22.298 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:22.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:21 smithi042 ceph-mon[108570]: pgmap v1462: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:06:22.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:21 smithi042 ceph-mon[113926]: pgmap v1462: 99 pgs: 99 active+clean; 5.5 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-13T15:06:22.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.435 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:22.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:22.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 58 16 95839 95823 103.239 64.6875 0.0100322 0.00968108 2024-09-13T15:06:22.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:22.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:22.676 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.716 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:22.820 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:22.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:22.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:22.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:22.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:22.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:23.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:23.119 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:23.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:23.220 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:23.242 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:23.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.379 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:23.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:23.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 59 16 96676 96660 102.376 52.3125 0.0164776 0.00976243 2024-09-13T15:06:23.481 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:23.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:23.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:23.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:23.657 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:23.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:23.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:23.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:23.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:23.902 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:23.936 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:23.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:23.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:24.038 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:24.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:24.153 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:23 smithi159 ceph-mon[102519]: pgmap v1463: 99 pgs: 99 active+clean; 5.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:06:24.157 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:23 smithi042 ceph-mon[108570]: pgmap v1463: 99 pgs: 99 active+clean; 5.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:06:24.157 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:23 smithi042 ceph-mon[113926]: pgmap v1463: 99 pgs: 99 active+clean; 5.8 GiB data, 25 GiB used, 600 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-13T15:06:24.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:24.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:24.303 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.337 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:24.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:06:24.471028+0000 min lat: 0.00110032 max lat: 0.302168 avg lat: 0.00981517 2024-09-13T15:06:24.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:06:24.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 60 16 97794 97778 101.834 69.875 0.00677645 0.00981517 2024-09-13T15:06:24.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:24.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:24.627 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:24.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:24.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:24.836 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:06:24.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:24.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:24.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:24.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:25.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:25.039 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:25.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:25.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:25.287 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:25.326 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:25.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:25.381 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:25.428 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:25.469 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:06:25.470 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:25.471 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 61 16 98740 98724 101.133 59.125 0.0131195 0.0098807 2024-09-13T15:06:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.508 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:25.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:25.521 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:06:25.521 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:25.561 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:06:25.561 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:25.598 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:06:25.598 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:25.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:25.627 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:06:25.627 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.662 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:25.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:25.663 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:06:25.663 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:25.705 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:06:25.706 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:25.749 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:06:25.750 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:25.763 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:25.813 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:25.915 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:25.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:25.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:25.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:26.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:26.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:25 smithi042 ceph-mon[108570]: pgmap v1464: 99 pgs: 99 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:06:26.054 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3718566357' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:26.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:25 smithi042 ceph-mon[113926]: pgmap v1464: 99 pgs: 99 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:06:26.054 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3718566357' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:26.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:26.183 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:26.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:26.223 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:25 smithi159 ceph-mon[102519]: pgmap v1464: 99 pgs: 99 active+clean; 5.8 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-13T15:06:26.223 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3718566357' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:26.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:26.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:26.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 62 16 99667 99651 100.437 57.9375 0.00805989 0.00995138 2024-09-13T15:06:26.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:26.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:26.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:26.614 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:26.642 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:26.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:26.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.781 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:26.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:26.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:26.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:26.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:26.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:27.007 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:27.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:27.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:27.165 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:27.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:27.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:27.416 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.456 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:27.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 63 16 100614 100598 99.7819 59.1875 0.00589921 0.0100161 2024-09-13T15:06:27.558 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:27.605 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:27.707 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.746 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:27.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:27.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:27.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:27.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:27.990 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:28.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:28.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:28.050 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:28.120 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:28.165 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[108570]: pgmap v1465: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T15:06:28.165 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.166 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.166 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[113926]: pgmap v1465: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T15:06:28.166 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.166 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:28 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.173 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:28.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:28.275 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:28.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:28.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:28.421 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:28.462 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:28.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 64 16 101611 101595 99.1964 62.3125 0.0272041 0.0100756 2024-09-13T15:06:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:28 smithi159 ceph-mon[102519]: pgmap v1465: 99 pgs: 99 active+clean; 6.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-13T15:06:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:28 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:28 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:28.561 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:06:28.562 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:28.562 INFO:tasks.thrashosds.thrasher:Reweighting osd 1 to 0.6762768975703097 2024-09-13T15:06:28.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 0.6762768975703097 2024-09-13T15:06:28.564 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:28.592 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:28.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:28.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.725 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:28.827 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:28.862 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:28.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:28.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:28.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:29.004 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:29.106 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:29.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:29.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:29.238 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:29.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:29.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:29.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:29.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 65 16 102671 102655 98.6895 66.25 0.00839447 0.0101275 2024-09-13T15:06:29.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:29.548 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:29.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:29.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:29.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:29.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.832 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:29.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:29.933 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:29.958 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:29.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:30.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:30.084 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:29] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.51.0" 2024-09-13T15:06:30.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:30.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:30.238 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:30.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:30.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:30.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:30 smithi042 ceph-mon[108570]: pgmap v1466: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:06:30.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:30 smithi042 ceph-mon[113926]: pgmap v1466: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:06:30.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:30.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:30.472 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 66 16 103768 103752 98.2328 68.5625 0.0177926 0.010175 2024-09-13T15:06:30.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:30 smithi159 ceph-mon[102519]: pgmap v1466: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-13T15:06:30.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:30.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:30.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:30.650 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:30.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:30.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:30.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:30.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:30.887 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:30.919 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:30.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:31.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.054 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:31.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:31.155 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:31.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:31.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:31.290 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:31.339 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:31.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:31.441 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:31.473 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 67 16 104771 104755 97.7023 62.6875 0.010581 0.0102307 2024-09-13T15:06:31.484 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:31.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:31.623 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:31.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:31.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:31.764 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:31.787 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:06:31.866 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:31.892 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:31.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:31.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:31.970 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:31.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:32.077 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:32.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:32.105 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:06:32.178 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:32.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.203 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:32.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:32.304 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:32.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:32 smithi042 ceph-mon[113926]: pgmap v1467: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T15:06:32.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:32 smithi042 ceph-mon[108570]: pgmap v1467: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T15:06:32.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:32.435 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:32.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:32.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 68 16 105733 105717 97.1458 60.125 0.019012 0.0102887 2024-09-13T15:06:32.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:32 smithi159 ceph-mon[102519]: pgmap v1467: 99 pgs: 99 active+clean; 6.1 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-13T15:06:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:06:32.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:32.612 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:32.715 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:32.745 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:32.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:32.883 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:32.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:32.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:33.029 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:33.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:33.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:33.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:33.297 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:33.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:33.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:33.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[113926]: osdmap e405: 8 total, 8 up, 7 in 2024-09-13T15:06:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:33.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:33 smithi042 ceph-mon[108570]: osdmap e405: 8 total, 8 up, 7 in 2024-09-13T15:06:33.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:33.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:33.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:33.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 69 16 106749 106733 96.6581 63.5 0.0987188 0.0103321 2024-09-13T15:06:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:33 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1014047592' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:06:33.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:33 smithi159 ceph-mon[102519]: osdmap e405: 8 total, 8 up, 7 in 2024-09-13T15:06:33.523 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:33.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:33.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:33.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.735 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:33.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:33.836 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:33.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:33.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:33.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:33.973 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:34.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:34.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:34.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.159 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:34.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.260 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:34.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:34.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:34.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:34 smithi042 ceph-mon[113926]: pgmap v1469: 99 pgs: 99 active+clean; 6.4 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-13T15:06:34.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:34 smithi042 ceph-mon[108570]: pgmap v1469: 99 pgs: 99 active+clean; 6.4 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-13T15:06:34.403 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:34.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:34.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:34.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 70 16 107809 107793 96.2237 66.25 0.0141294 0.0103872 2024-09-13T15:06:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:34 smithi159 ceph-mon[102519]: pgmap v1469: 99 pgs: 99 active+clean; 6.4 GiB data, 27 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-13T15:06:34.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.659 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:34.690 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:34.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:34.792 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:34.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.830 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:34.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:34.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:34.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:34.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:34.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:35.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:35.127 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:35.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:35.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:35.361 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:35.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:35.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 71 16 108835 108819 95.7716 64.125 0.00718044 0.0104368 2024-09-13T15:06:35.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:35.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:35.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:35.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:35.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:35.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:35.902 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:06:35.902 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:06:35.902 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:06:35.902 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:06:35.902 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:06:35.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:35.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:35.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:36.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:36.122 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:36.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:36.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:36.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:36.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:36.409 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:36.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:36.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 72 15 109886 109871 95.3546 65.75 0.0157931 0.010482 2024-09-13T15:06:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:36 smithi159 ceph-mon[102519]: pgmap v1470: 99 pgs: 99 active+clean; 6.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:06:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:36 smithi042 ceph-mon[108570]: pgmap v1470: 99 pgs: 99 active+clean; 6.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:06:36.499 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:36 smithi042 ceph-mon[113926]: pgmap v1470: 99 pgs: 99 active+clean; 6.4 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2024-09-13T15:06:36.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:36.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:36.566 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:36.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:36.668 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:36.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:36.804 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:36.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:36.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:36.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:36.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:06:37.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:37.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.020 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:37.106 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:37.144 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:37.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:37.246 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:37.270 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:37.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:37.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:37.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:37.475 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 73 16 110884 110868 94.9017 62.3125 0.00945495 0.0105321 2024-09-13T15:06:37.508 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:37.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:37.709 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:37.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:37.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:37.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:37.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:37.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:37.988 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:37.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:37.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:38.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:38.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:38.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:38.234 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:38.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.276 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:38.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:38.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:38 smithi042 ceph-mon[108570]: pgmap v1471: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T15:06:38.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:38 smithi042 ceph-mon[113926]: pgmap v1471: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T15:06:38.376 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:38.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:38.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:38.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 74 16 111923 111907 94.4968 64.9375 0.0104053 0.0105773 2024-09-13T15:06:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:38 smithi159 ceph-mon[102519]: pgmap v1471: 99 pgs: 99 active+clean; 6.6 GiB data, 28 GiB used, 598 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-13T15:06:38.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:38.537 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:06:38.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:38.651 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:38.677 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:38.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:38.779 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.824 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:38.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:38.925 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:38.970 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:38.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:39.072 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:39.098 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.200 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:39.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.326 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:39.357 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.459 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:39.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 75 16 112863 112847 94.0201 58.75 0.0208859 0.0106308 2024-09-13T15:06:39.490 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:39.592 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:39.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:39.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:39.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:39.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:39] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-09-13T15:06:39.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:39.884 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:39.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:39.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:40.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:40.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:40.050 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:40.123 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 0.676277 (ad20) 2024-09-13T15:06:40.123 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:06:40.124 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:40.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:40.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:40.158 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:40.179 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[108570]: pgmap v1472: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:06:40.179 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097} : dispatch 2024-09-13T15:06:40.179 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3185460125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:40.180 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[113926]: pgmap v1472: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:06:40.180 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097} : dispatch 2024-09-13T15:06:40.180 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:40 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3185460125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:40.181 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:06:40.181 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:40.201 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:06:40.201 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:40.222 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:06:40.223 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:40.239 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:06:40.239 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:40.255 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:06:40.255 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:40.257 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:40.272 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:06:40.272 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:40.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:40.288 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:06:40.288 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:40.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.405 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:40.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 76 16 113509 113493 93.3143 40.375 0.0036498 0.0106669 2024-09-13T15:06:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:40 smithi159 ceph-mon[102519]: pgmap v1472: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-13T15:06:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097} : dispatch 2024-09-13T15:06:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:40 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3185460125' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:40.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:40.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:40.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:40.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:40.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:40.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:40.755 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:40.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.778 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:40.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:40.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:40.900 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:40.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:40.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:41.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:41.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:41.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:41.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:41.151 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:41.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:41.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:41.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:41.377 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:41.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:41.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 77 16 113509 113493 92.1025 0 - 0.0106669 2024-09-13T15:06:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097}]': finished 2024-09-13T15:06:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:41 smithi159 ceph-mon[102519]: osdmap e406: 8 total, 8 up, 7 in 2024-09-13T15:06:41.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:41 smithi159 ceph-mon[102519]: osdmap e407: 8 total, 8 up, 7 in 2024-09-13T15:06:41.500 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:41.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097}]': finished 2024-09-13T15:06:41.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[113926]: osdmap e406: 8 total, 8 up, 7 in 2024-09-13T15:06:41.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[113926]: osdmap e407: 8 total, 8 up, 7 in 2024-09-13T15:06:41.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1916476290' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.6762768975703097}]': finished 2024-09-13T15:06:41.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[108570]: osdmap e406: 8 total, 8 up, 7 in 2024-09-13T15:06:41.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:41 smithi042 ceph-mon[108570]: osdmap e407: 8 total, 8 up, 7 in 2024-09-13T15:06:41.624 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:41.652 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:41.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:41.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:41.777 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:41.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:41.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:41.910 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:41.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:42.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:42.142 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:42.175 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:42.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:42.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:42.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:42.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:42.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.449 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:42.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:42.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 78 16 114000 113984 91.3152 15.3438 0.00561802 0.0109452 2024-09-13T15:06:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:42 smithi159 ceph-mon[102519]: pgmap v1475: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s 2024-09-13T15:06:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:42 smithi159 ceph-mon[102519]: osdmap e408: 8 total, 8 up, 7 in 2024-09-13T15:06:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:42 smithi159 ceph-mon[102519]: 14.8 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7057 2024-09-13T15:06:42.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:42.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[113926]: pgmap v1475: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s 2024-09-13T15:06:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[113926]: osdmap e408: 8 total, 8 up, 7 in 2024-09-13T15:06:42.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[113926]: 14.8 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7057 2024-09-13T15:06:42.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[108570]: pgmap v1475: 99 pgs: 99 active+clean; 6.7 GiB data, 28 GiB used, 597 GiB / 626 GiB avail; 48 MiB/s wr, 767 op/s 2024-09-13T15:06:42.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[108570]: osdmap e408: 8 total, 8 up, 7 in 2024-09-13T15:06:42.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:42 smithi042 ceph-mon[108570]: 14.8 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7057 2024-09-13T15:06:42.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:42.716 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:42.818 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:42.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:42.959 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:42.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:42.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.090 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.224 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:43.266 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:43.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:43.368 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:43.403 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:43.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 79 16 114922 114906 90.8887 57.625 0.00448639 0.0109944 2024-09-13T15:06:43.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:43 smithi159 ceph-mon[102519]: 14.3 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7205 2024-09-13T15:06:43.505 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.545 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:43 smithi042 ceph-mon[113926]: 14.3 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7205 2024-09-13T15:06:43.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:43 smithi042 ceph-mon[108570]: 14.3 starting backfill to osd.5 from (0'0,0'0] MAX to 405'7205 2024-09-13T15:06:43.647 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:43.683 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:43.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.828 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:43.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:43.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:43.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:44.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.114 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:44.216 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:44.256 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:44.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:44 smithi042 ceph-mon[113926]: pgmap v1477: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 899 op/s; 7142/339072 objects misplaced (2.106%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:06:44.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:44 smithi042 ceph-mon[108570]: pgmap v1477: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 899 op/s; 7142/339072 objects misplaced (2.106%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:06:44.358 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:44.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:44.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:2024-09-13T15:06:44.476056+0000 min lat: 0.00110032 max lat: 1.91486 avg lat: 0.0110444 2024-09-13T15:06:44.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-13T15:06:44.476 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 80 16 115861 115845 90.4861 58.6875 0.00561067 0.0110444 2024-09-13T15:06:44.496 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:44 smithi159 ceph-mon[102519]: pgmap v1477: 99 pgs: 1 active+remapped+backfilling, 98 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 56 MiB/s wr, 899 op/s; 7142/339072 objects misplaced (2.106%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:06:44.532 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:44.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:44.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:44.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:44.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:44.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:44.811 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:44.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:44.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:44.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:44.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:44.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:45.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:45.093 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:45.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:45.195 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:45.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:45.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:45.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.364 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:45.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:45.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:45.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 81 16 116789 116773 90.0842 58 0.0211949 0.0110946 2024-09-13T15:06:45.483 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:45 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 1/341583 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:45.483 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:45 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 1/341583 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:45.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:45 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 1/341583 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:45.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.519 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:45.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:45.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:45.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:45.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:45.666 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:06:45.761 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:45.786 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:45.888 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:45.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:45.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:46.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:46.081 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:46.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:46.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:46.188 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:06:46.188 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:06:46.221 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:46.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:46.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:46.461 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 82 16 117624 117608 89.622 52.1875 0.00830667 0.0111518 2024-09-13T15:06:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:46 smithi159 ceph-mon[102519]: pgmap v1478: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 1/341583 objects degraded (0.000%); 21450/341583 objects misplaced (6.280%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:46.500 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:46.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:46 smithi042 ceph-mon[108570]: pgmap v1478: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 1/341583 objects degraded (0.000%); 21450/341583 objects misplaced (6.280%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:46.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:46 smithi042 ceph-mon[113926]: pgmap v1478: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 6.9 GiB data, 29 GiB used, 597 GiB / 626 GiB avail; 35 MiB/s wr, 561 op/s; 1/341583 objects degraded (0.000%); 21450/341583 objects misplaced (6.280%); 1.2 MiB/s, 18 objects/s recovering 2024-09-13T15:06:46.602 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:46.645 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:46.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:46.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:46.792 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:46.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:46.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:46.925 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:46.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:46.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.027 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:47.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:47.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.064 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:47.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:47.201 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:47.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:47.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:47.338 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:47.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:47.440 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:47.473 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:47.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:47.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 83 16 118516 118500 89.2139 55.75 0.00740243 0.0112025 2024-09-13T15:06:47.575 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.607 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:47.754 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:47.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:47.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:47.890 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:47.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:47.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:47.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.047 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:48.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:48.148 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.183 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:48.285 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.319 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:48.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:48.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:48.449 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:48.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 84 16 119360 119344 88.7798 52.75 0.00430951 0.0112548 2024-09-13T15:06:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:48 smithi159 ceph-mon[102519]: pgmap v1479: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 7.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 1/347187 objects degraded (0.000%); 21450/347187 objects misplaced (6.178%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:48 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:48 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:48.583 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:48.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[108570]: pgmap v1479: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 7.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 1/347187 objects degraded (0.000%); 21450/347187 objects misplaced (6.178%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[113926]: pgmap v1479: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfilling, 96 active+clean; 7.1 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s; 1/347187 objects degraded (0.000%); 21450/347187 objects misplaced (6.178%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:06:48.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:48 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:48.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:48.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:48.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:48.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:48.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:48.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:48.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:48.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:49.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:06:49.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:49.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:49.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:49.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:49.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:49.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:49.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:49.477 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 85 16 120126 120110 88.2986 47.875 0.00680463 0.0113186 2024-09-13T15:06:49.488 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.528 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:49.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:49.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:49.674 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:49.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:49.776 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:49.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:49.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:49.900 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:49.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:49.933 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:49.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:50.034 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.065 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:50.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:50.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:49] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.51.0" 2024-09-13T15:06:50.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.206 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:50.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:50.308 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:50.343 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:50.445 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:50.478 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 86 16 121004 120988 87.9099 54.875 0.00871228 0.0113683 2024-09-13T15:06:50.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.479 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:50.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:50 smithi159 ceph-mon[102519]: pgmap v1480: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T15:06:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:50.580 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[113926]: pgmap v1480: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T15:06:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:50.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:50.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:50.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[108570]: pgmap v1480: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 8.2 MiB/s, 131 objects/s recovering 2024-09-13T15:06:50.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:06:50.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:06:50.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:50.602 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:50.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:50.703 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:50.751 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:50.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:50.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:50.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:50.976 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:51.016 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:51.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:51.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:51.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:51.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:51.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:51.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:51.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:51.337 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:51 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1/354660 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:51.337 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:51.337 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:51.394 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:51.430 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:51.478 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 87 15 121718 121703 87.4131 44.6875 0.0149654 0.0114349 2024-09-13T15:06:51.532 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:51.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1/354660 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:51.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1/354660 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:06:51.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:06:51.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:51.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:51.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:51.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:51.830 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:51.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:51.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:51.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:52.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.110 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:52.174 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:52.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:52.242 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:52.320 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 5, 6, 7, 4] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:06:52.320 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:06:52.320 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 2, 3, 5, 6, 7, 4] 2024-09-13T15:06:52.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd out 1 2024-09-13T15:06:52.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:52 smithi042 ceph-mon[108570]: pgmap v1481: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T15:06:52.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:52 smithi042 ceph-mon[113926]: pgmap v1481: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T15:06:52.344 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:52.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:52.477 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:52.479 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:52 smithi159 ceph-mon[102519]: pgmap v1481: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.2 GiB data, 30 GiB used, 596 GiB / 626 GiB avail; 48 MiB/s wr, 772 op/s; 1/354660 objects degraded (0.000%); 21017/354660 objects misplaced (5.926%); 6.7 MiB/s, 107 objects/s recovering 2024-09-13T15:06:52.480 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 88 16 122470 122454 86.9509 46.9375 0.033914 0.0114954 2024-09-13T15:06:52.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:52.642 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:52.682 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:52.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:52.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:52.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:52.937 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:52.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:52.969 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:52.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:53.070 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:53.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:53.199 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:53.225 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:53.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:53.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:53.363 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:53.465 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:53.480 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 89 16 123271 123255 86.5361 50.0625 0.0259558 0.0115498 2024-09-13T15:06:53.497 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:53.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:53.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:53.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.641 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:53.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:53.742 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:53.776 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:53.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:53.878 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:53.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:53.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:54.017 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.051 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:54.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:54.178 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:54.280 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:54.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:54.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:54.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:54.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:54.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:54.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout: 90 1 123954 123953 86.0594 43.625 0.0217193 0.0116139 2024-09-13T15:06:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:54 smithi159 ceph-mon[102519]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.4 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s; 1/363729 objects degraded (0.000%); 20478/363729 objects misplaced (5.630%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:06:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3077160244' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:54 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:54.565 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:54.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:54.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[108570]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.4 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s; 1/363729 objects degraded (0.000%); 20478/363729 objects misplaced (5.630%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:06:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3077160244' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[113926]: pgmap v1482: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.4 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 826 op/s; 1/363729 objects degraded (0.000%); 20478/363729 objects misplaced (5.630%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:06:54.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3077160244' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:54 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:06:54.667 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:54.695 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:54.797 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:54.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:54.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:54.923 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:54.948 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:54.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:55.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:55.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:55.178 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.304 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:55.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:55.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:55.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[108570]: osdmap e409: 8 total, 8 up, 7 in 2024-09-13T15:06:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:55 smithi042 ceph-mon[113926]: osdmap e409: 8 total, 8 up, 7 in 2024-09-13T15:06:55.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:55.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total time run: 90.0236 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Total writes made: 123954 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Write size: 65536 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Object size: 65536 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Bandwidth (MB/sec): 86.0566 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Bandwidth: 38.77 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max bandwidth (MB/sec): 238.562 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min bandwidth (MB/sec): 0 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average IOPS: 1376 2024-09-13T15:06:55.481 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev IOPS: 620.329 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max IOPS: 3817 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min IOPS: 0 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Average Latency(s): 0.0116141 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Stddev Latency(s): 0.0230177 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Max latency(s): 1.91486 2024-09-13T15:06:55.482 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Min latency(s): 0.00110032 2024-09-13T15:06:55.487 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Cleaning up (deleting benchmark objects) 2024-09-13T15:06:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:55 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:06:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:55 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:06:55.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:55 smithi159 ceph-mon[102519]: osdmap e409: 8 total, 8 up, 7 in 2024-09-13T15:06:55.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:55.592 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:55.647 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:06:55.694 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:55.736 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:55.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:55.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:55.878 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:55.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:55.915 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:55.979 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:56.026 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:56.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:56.169 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:56.271 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:56.299 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:56.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:56.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:56.420 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:06:56.429 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:56.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:56.531 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:56.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:56.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:56 smithi042 ceph-mon[108570]: pgmap v1484: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 1/366768 objects degraded (0.000%); 20686/366768 objects misplaced (5.640%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:56.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1/366768 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:56.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:56 smithi042 ceph-mon[113926]: pgmap v1484: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 1/366768 objects degraded (0.000%); 20686/366768 objects misplaced (5.640%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:56.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1/366768 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:56.675 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:56.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:56.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:56.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:56 smithi159 ceph-mon[102519]: pgmap v1484: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 52 MiB/s wr, 839 op/s; 1/366768 objects degraded (0.000%); 20686/366768 objects misplaced (5.640%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:56.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1/366768 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:06:56.822 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:56.875 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:56.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:56.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:57.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:57.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:57.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:57.143 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:57.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:57.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.286 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:57.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:57.388 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:57.421 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:57.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:57.523 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:57.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:57.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.696 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:57.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.833 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:57.935 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:06:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:57.970 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:57.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:06:58.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:58.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:58.214 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.230 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:06:58.230 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:06:58.262 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:58.273 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:06:58.273 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:06:58.309 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:06:58.310 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:06:58.350 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:06:58.351 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:06:58.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:58.381 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:06:58.382 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:06:58.383 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:06:58.389 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:58.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:06:58.416 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:06:58.416 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:06:58.464 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:06:58.464 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:06:58.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:58.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:58 smithi042 ceph-mon[108570]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 791 op/s; 1/368475 objects degraded (0.000%); 20686/368475 objects misplaced (5.614%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:58.528 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:06:58 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3428464164' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:58.529 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:58 smithi042 ceph-mon[113926]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 791 op/s; 1/368475 objects degraded (0.000%); 20686/368475 objects misplaced (5.614%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:58.529 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:06:58 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3428464164' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:58.537 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:06:58.537 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:06:58.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:58.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:58.652 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:58.686 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:58.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:58 smithi159 ceph-mon[102519]: pgmap v1485: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 47 MiB/s wr, 791 op/s; 1/368475 objects degraded (0.000%); 20686/368475 objects misplaced (5.614%); 11 MiB/s, 176 objects/s recovering 2024-09-13T15:06:58.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:06:58 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3428464164' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:06:58.788 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:06:58.826 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:06:58.928 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:06:58.968 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:58.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:58.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:06:59.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:59.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.218 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:06:59.255 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:59.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:06:59.357 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.390 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:59.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:59.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:06:59.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:06:59.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:06:59.630 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:06:59.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:06:59.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:06:59.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:59.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:06:59.907 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:06:59.937 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:06:59.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:06:59.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:00.039 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:00.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.079 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:06:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:06:59] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-13T15:07:00.180 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:00.231 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:00.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.369 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:00.402 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:00 smithi042 ceph-mon[108570]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:00.404 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:00 smithi042 ceph-mon[113926]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:00.470 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:00.508 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:00.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:00.653 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:00 smithi159 ceph-mon[102519]: pgmap v1486: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:00.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:00.793 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:00.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:00.895 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:00.928 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:00.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:01.030 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:01.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:01.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:01.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:01.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:01.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:01.336 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:01.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:01.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.475 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:01.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:01.576 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:01.583 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:01 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1/367461 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:01.583 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:01 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1/367461 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:01.615 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:01.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:01.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:01.736 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:01 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1/367461 objects degraded (0.000%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:01.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:01.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:01 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:07:01.581+0000 7fcad2a32640 -1 mon.a@0(leader).osd e409 definitely_dead 0 2024-09-13T15:07:01.853 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:01.889 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:01.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:01.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:01.991 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:02.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:02.121 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:02.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:02.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:02.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:02.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:02.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.402 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:02.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:02.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:02.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[108570]: pgmap v1487: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:02.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1325943885' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[108570]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[113926]: pgmap v1487: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1325943885' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:02 smithi042 ceph-mon[113926]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T15:07:02.631 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:02.652 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:02.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:02 smithi159 ceph-mon[102519]: pgmap v1487: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 7.5 GiB data, 31 GiB used, 595 GiB / 626 GiB avail; 102 B/s rd, 36 MiB/s wr, 725 op/s; 1/367461 objects degraded (0.000%); 19067/367461 objects misplaced (5.189%); 15 MiB/s, 236 objects/s recovering 2024-09-13T15:07:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:02 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1325943885' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:02 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-09-13T15:07:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:02 smithi159 ceph-mon[102519]: Client client.admin marked osd.1 out, while it was still marked up 2024-09-13T15:07:02.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:02.778 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:02.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:02.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:02.902 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:02.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:02.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:03.003 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:03.025 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:03.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:03.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:03.151 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:03.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:03.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:03.275 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:03.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:03.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:03.401 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:03.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:03.503 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:03.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:03.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T15:07:03.614 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[113926]: osdmap e410: 8 total, 8 up, 6 in 2024-09-13T15:07:03.615 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[113926]: osdmap e411: 8 total, 8 up, 6 in 2024-09-13T15:07:03.615 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T15:07:03.615 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[108570]: osdmap e410: 8 total, 8 up, 6 in 2024-09-13T15:07:03.615 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:03 smithi042 ceph-mon[108570]: osdmap e411: 8 total, 8 up, 6 in 2024-09-13T15:07:03.628 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:03.649 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-09-13T15:07:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:03 smithi159 ceph-mon[102519]: osdmap e410: 8 total, 8 up, 6 in 2024-09-13T15:07:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:03 smithi159 ceph-mon[102519]: osdmap e411: 8 total, 8 up, 6 in 2024-09-13T15:07:03.752 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:03.774 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:03.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:03.876 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:03.898 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:03.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:03.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:03.999 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:04.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:04.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.147 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:04.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:04.272 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:04.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:04.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:04.401 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:04.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:04.502 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:04.526 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:04.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:04.628 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:04.650 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:04.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:04.720 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[108570]: pgmap v1489: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 9 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 7.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 121 B/s rd, 13 MiB/s wr, 777 op/s; 1/357405 objects degraded (0.000%); 17997/357405 objects misplaced (5.035%); 18 MiB/s, 292 objects/s recovering 2024-09-13T15:07:04.720 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[113926]: pgmap v1489: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 9 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 7.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 121 B/s rd, 13 MiB/s wr, 777 op/s; 1/357405 objects degraded (0.000%); 17997/357405 objects misplaced (5.035%); 18 MiB/s, 292 objects/s recovering 2024-09-13T15:07:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:04 smithi159 ceph-mon[102519]: pgmap v1489: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 9 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 7.3 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 121 B/s rd, 13 MiB/s wr, 777 op/s; 1/357405 objects degraded (0.000%); 17997/357405 objects misplaced (5.035%); 18 MiB/s, 292 objects/s recovering 2024-09-13T15:07:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:04 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:04 smithi159 ceph-mon[102519]: osdmap e412: 8 total, 8 up, 6 in 2024-09-13T15:07:04.752 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:04.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.787 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:04.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:04.888 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:04.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:04.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:05.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:05.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[113926]: osdmap e412: 8 total, 8 up, 6 in 2024-09-13T15:07:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:04 smithi042 ceph-mon[108570]: osdmap e412: 8 total, 8 up, 6 in 2024-09-13T15:07:05.159 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:05.196 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:05.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:05.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:05.331 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:05.433 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:05.471 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:07:05.471 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:07:05.472 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.472 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129899 14.b 14:da4c7396:::benchmark_data_smithi042_319156_object5942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:07:05.472 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:07:05.472 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.3972336350000001, 2024-09-13T15:07:05.472 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3972583599999999, 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for peered", 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:05.473 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129899 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327271+0000", 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.468e-06 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327277+0000", 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2279999999999998e-06 2024-09-13T15:07:05.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.328798+0000", 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015211020000000001 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:05.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.328813+0000", 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4973999999999999e-05 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331493+0000", 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026804400000000001 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331612+0000", 2024-09-13T15:07:05.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000118748 2024-09-13T15:07:05.479 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.479 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:07:05.479 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331726+0000", 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011390199999999999 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351419+0000", 2024-09-13T15:07:05.480 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019692949000000001 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351430+0000", 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1068e-05 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330782+0000", 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979351839 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330794+0000", 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2201000000000001e-05 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129942 14.b 14.156bb53b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.3379324260000001, 2024-09-13T15:07:05.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3380121709999999, 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for peered", 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129942 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:05.483 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386572+0000", 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2699999999999999e-07 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386576+0000", 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.211e-06 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:05.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386587+0000", 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0947999999999999e-05 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386594+0000", 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2590000000000003e-06 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386667+0000", 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3158999999999995e-05 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.486 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386677+0000", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4499999999999993e-06 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330921+0000", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94424369399999997 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330929+0000", 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4279999999999996e-06 2024-09-13T15:07:05.487 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129914 14.b 14:d5295360:::benchmark_data_smithi042_319156_object5957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.3843432369999999, 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3844614710000001, 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for peered", 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:05.488 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129914 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:07:05.489 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340162+0000", 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9609999999999999e-06 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340170+0000", 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5519999999999999e-06 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340182+0000", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2037e-05 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340191+0000", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8230000000000001e-06 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340293+0000", 2024-09-13T15:07:05.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000102279 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340441+0000", 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014848899999999999 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340563+0000", 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122058 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351455+0000", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010891675999999999 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351462+0000", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8750000000000002e-06 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330879+0000", 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97941681000000003 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330890+0000", 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.164e-05 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129950 14.b 14.72b5405b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 1.3019228380000001, 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.302084636, 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:05.494 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for peered", 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129950 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422582+0000", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.491e-06 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422588+0000", 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0390000000000001e-06 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422602+0000", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4138000000000001e-05 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422610+0000", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3200000000000002e-06 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422668+0000", 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7917999999999997e-05 2024-09-13T15:07:05.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422676+0000", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9699999999999999e-06 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331166+0000", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90849047500000002 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331175+0000", 2024-09-13T15:07:05.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8559999999999996e-06 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "MOSDPGPushReply(14.6 411/407 [PushReplyOp(14:6517707e:::benchmark_data_smithi042_319156_object7224:head)])", 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:03.720280+0000", 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0042232859999999997, 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0044199449999999998, 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:05.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720280+0000", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720280+0000", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720281+0000", 2024-09-13T15:07:05.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3900000000000004e-07 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720286+0000", 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5909999999999998e-06 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720295+0000", 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6180000000000001e-06 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:05.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720302+0000", 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7689999999999999e-06 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.720363+0000", 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0602e-05 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 5 2024-09-13T15:07:05.502 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:07:05.540 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:05.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:05.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:05 smithi159 ceph-mon[102519]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 409'8184 2024-09-13T15:07:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:05 smithi159 ceph-mon[102519]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 409'8293 2024-09-13T15:07:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:05 smithi159 ceph-mon[102519]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 409'8174 2024-09-13T15:07:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:05 smithi159 ceph-mon[102519]: 14.f starting backfill to osd.5 from (0'0,0'0] MAX to 409'8078 2024-09-13T15:07:05.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:05 smithi159 ceph-mon[102519]: osdmap e413: 8 total, 8 up, 6 in 2024-09-13T15:07:05.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:05.809 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:05.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[113926]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 409'8184 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[113926]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 409'8293 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[113926]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 409'8174 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[113926]: 14.f starting backfill to osd.5 from (0'0,0'0] MAX to 409'8078 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[113926]: osdmap e413: 8 total, 8 up, 6 in 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[108570]: 14.1 starting backfill to osd.6 from (0'0,0'0] MAX to 409'8184 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[108570]: 14.a starting backfill to osd.4 from (0'0,0'0] MAX to 409'8293 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[108570]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 409'8174 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[108570]: 14.f starting backfill to osd.5 from (0'0,0'0] MAX to 409'8078 2024-09-13T15:07:05.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:05 smithi042 ceph-mon[108570]: osdmap e413: 8 total, 8 up, 6 in 2024-09-13T15:07:05.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:05.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:05.944 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:05.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:06.044 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:06.080 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:06.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.216 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:06.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:06.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:06.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:06.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:06.493 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:06.594 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:06.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:06.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: 14.b starting backfill to osd.7 from (0'0,0'0] MAX to 409'8087 2024-09-13T15:07:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: pgmap v1492: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 78 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 78/354777 objects degraded (0.022%); 17617/354777 objects misplaced (4.966%); 10 MiB/s, 166 objects/s recovering 2024-09-13T15:07:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 78/354777 objects degraded (0.022%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:06.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:06 smithi159 ceph-mon[102519]: osdmap e414: 8 total, 8 up, 6 in 2024-09-13T15:07:06.766 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:06.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:06.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: 14.b starting backfill to osd.7 from (0'0,0'0] MAX to 409'8087 2024-09-13T15:07:06.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: pgmap v1492: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 78 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 78/354777 objects degraded (0.022%); 17617/354777 objects misplaced (4.966%); 10 MiB/s, 166 objects/s recovering 2024-09-13T15:07:06.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 78/354777 objects degraded (0.022%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:06.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:06.770 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[113926]: osdmap e414: 8 total, 8 up, 6 in 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: 14.b starting backfill to osd.7 from (0'0,0'0] MAX to 409'8087 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: pgmap v1492: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 78 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 78/354777 objects degraded (0.022%); 17617/354777 objects misplaced (4.966%); 10 MiB/s, 166 objects/s recovering 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 78/354777 objects degraded (0.022%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:06.771 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:06 smithi042 ceph-mon[108570]: osdmap e414: 8 total, 8 up, 6 in 2024-09-13T15:07:06.868 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:06.904 INFO:teuthology.orchestra.run.smithi042.stderr:marked out osd.1. 2024-09-13T15:07:06.905 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:06.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:06.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:07.007 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:07.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:07.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:07.179 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:07.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:07.281 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:07.313 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:07.415 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:07.450 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:07.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:07.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:07.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.689 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.720 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.822 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:07.856 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:07.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:07.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:07.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:08.005 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:08.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:08.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:07:08.106 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:08.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:08.236 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:08.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:08.266 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:08.368 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:08.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:08.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:08.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:08.556 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:08.581 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:08 smithi042 ceph-mon[108570]: pgmap v1495: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 476 op/s; 78/351849 objects degraded (0.022%); 24906/351849 objects misplaced (7.079%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:08.581 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:08 smithi042 ceph-mon[113926]: pgmap v1495: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 476 op/s; 78/351849 objects degraded (0.022%); 24906/351849 objects misplaced (7.079%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:08.658 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:08.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:08.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:08 smithi159 ceph-mon[102519]: pgmap v1495: 99 pgs: 1 unknown, 1 active+remapped+backfilling, 17 peering, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 77 active+clean; 7.2 GiB data, 26 GiB used, 511 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 476 op/s; 78/351849 objects degraded (0.022%); 24906/351849 objects misplaced (7.079%); 1.4 MiB/s, 22 objects/s recovering 2024-09-13T15:07:08.793 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:08.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:07:08.830 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:08.933 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:08.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:08.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:09.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:09.111 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:09.146 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:09.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:09.251 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:09.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:09.383 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:09.485 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:09.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:09.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:09 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T15:07:09.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:09 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T15:07:09.615 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:09.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:09 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 6 pgs peering) 2024-09-13T15:07:09.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:09.799 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:09.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:09.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:09.932 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:09.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:10.033 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:10.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:10.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:09] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-13T15:07:10.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:10.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:10.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:10.339 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:10.441 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.475 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:10.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:10.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:10.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:10.645 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:10 smithi042 ceph-mon[113926]: pgmap v1496: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 481 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 25 MiB/s, 394 objects/s recovering 2024-09-13T15:07:10.645 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:10 smithi042 ceph-mon[108570]: pgmap v1496: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 481 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 25 MiB/s, 394 objects/s recovering 2024-09-13T15:07:10.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:10.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:10 smithi159 ceph-mon[102519]: pgmap v1496: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 481 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 25 MiB/s, 394 objects/s recovering 2024-09-13T15:07:10.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:10.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:10.852 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:10.884 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:10.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:10.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:10.986 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:11.022 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:11.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:11.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:11.262 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:11.297 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:11.399 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:11.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:11.536 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.563 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:11.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:11.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:11.705 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:11.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:11 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 163/347100 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:11.807 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:11.835 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:11.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:11 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 163/347100 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:11.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:11 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 163/347100 objects degraded (0.047%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:11.937 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:11.970 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:11.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:12.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:12.111 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:12.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:12.213 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:12.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:12.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:12.353 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.397 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:12.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:12.485 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:12.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:12.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:12.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:12.656 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:12.664 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:12 smithi042 ceph-mon[113926]: pgmap v1497: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 406 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 21 MiB/s, 333 objects/s recovering 2024-09-13T15:07:12.666 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:12 smithi042 ceph-mon[108570]: pgmap v1497: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 406 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 21 MiB/s, 333 objects/s recovering 2024-09-13T15:07:12.696 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:12.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:12.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:12 smithi159 ceph-mon[102519]: pgmap v1497: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 7.1 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 406 op/s; 163/347100 objects degraded (0.047%); 51364/347100 objects misplaced (14.798%); 21 MiB/s, 333 objects/s recovering 2024-09-13T15:07:12.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:12.847 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:12.949 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:12.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:12.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:12.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:13.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:13.121 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:13.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:13.222 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:13.252 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:13.267 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4] out_osds: [0, 1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:07:13.267 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:07:13.267 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-13T15:07:13.267 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-13T15:07:13.267 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-13T15:07:13.268 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-13T15:07:13.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:13.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:13.519 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:13.555 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:13.657 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:13.689 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:13.791 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:13.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:13.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:13.966 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:13.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:14.067 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:14.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.104 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:14.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:14.205 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:14.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:14.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:14.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:14.377 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:14.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:14.452 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:07:14.479 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:14.517 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:14.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:14.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:14.656 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:14 smithi159 ceph-mon[102519]: pgmap v1498: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 765 op/s; 163/336561 objects degraded (0.048%); 49347/336561 objects misplaced (14.662%); 23 MiB/s, 370 objects/s recovering 2024-09-13T15:07:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1415067151' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.758 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:14.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:14.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:14.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[108570]: pgmap v1498: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 765 op/s; 163/336561 objects degraded (0.048%); 49347/336561 objects misplaced (14.662%); 23 MiB/s, 370 objects/s recovering 2024-09-13T15:07:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1415067151' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[113926]: pgmap v1498: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 511 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 765 op/s; 163/336561 objects degraded (0.048%); 49347/336561 objects misplaced (14.662%); 23 MiB/s, 370 objects/s recovering 2024-09-13T15:07:14.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1415067151' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:14.913 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:14.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:14.951 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:14.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:15.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:15.084 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:15.186 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.221 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:15.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:15.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:15.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:15.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:15.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:15.502 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:15.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:15.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:15.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[113926]: osdmap e415: 8 total, 8 up, 6 in 2024-09-13T15:07:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:15.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:15 smithi042 ceph-mon[108570]: osdmap e415: 8 total, 8 up, 6 in 2024-09-13T15:07:15.603 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.635 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:15.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:15.737 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:15 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:07:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:15.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:15 smithi159 ceph-mon[102519]: osdmap e415: 8 total, 8 up, 6 in 2024-09-13T15:07:15.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.771 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:15.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:15.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:15.900 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:15.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:15.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:16.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:16.025 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:16.127 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:16.150 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:16.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:16.376 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:16.399 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:16.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:16.531 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:16.633 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:16.665 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:16.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:16 smithi159 ceph-mon[102519]: pgmap v1500: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 8.9 KiB/s rd, 26 KiB/s wr, 859 op/s; 377/333231 objects degraded (0.113%); 48517/333231 objects misplaced (14.560%); 22 MiB/s, 360 objects/s recovering 2024-09-13T15:07:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:16 smithi159 ceph-mon[102519]: osdmap e416: 8 total, 8 up, 6 in 2024-09-13T15:07:16.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 377/333231 objects degraded (0.113%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[108570]: pgmap v1500: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 8.9 KiB/s rd, 26 KiB/s wr, 859 op/s; 377/333231 objects degraded (0.113%); 48517/333231 objects misplaced (14.560%); 22 MiB/s, 360 objects/s recovering 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[108570]: osdmap e416: 8 total, 8 up, 6 in 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 377/333231 objects degraded (0.113%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[113926]: pgmap v1500: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 91 active+clean; 6.8 GiB data, 25 GiB used, 512 GiB / 536 GiB avail; 8.9 KiB/s rd, 26 KiB/s wr, 859 op/s; 377/333231 objects degraded (0.113%); 48517/333231 objects misplaced (14.560%); 22 MiB/s, 360 objects/s recovering 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[113926]: osdmap e416: 8 total, 8 up, 6 in 2024-09-13T15:07:16.767 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 377/333231 objects degraded (0.113%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:16.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:16.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:16.904 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:16.946 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:16.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:16.949 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:17.048 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.096 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:17.198 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.236 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:17.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:17.321 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:07:17.337 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:17.351 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:07:17.351 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:07:17.372 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:17.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:17.385 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:07:17.385 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:07:17.425 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:07:17.425 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:07:17.464 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:07:17.464 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:07:17.474 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:17.509 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:07:17.509 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:07:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.533 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:17.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:17.543 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:07:17.543 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:07:17.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:17 smithi042 ceph-mon[113926]: osdmap e417: 8 total, 8 up, 6 in 2024-09-13T15:07:17.544 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:17 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2855630288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:17.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:17 smithi042 ceph-mon[108570]: osdmap e417: 8 total, 8 up, 6 in 2024-09-13T15:07:17.546 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:17 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2855630288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:17.580 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:07:17.580 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:07:17.614 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:07:17.614 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:07:17.634 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:17.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:17 smithi159 ceph-mon[102519]: osdmap e417: 8 total, 8 up, 6 in 2024-09-13T15:07:17.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:17 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2855630288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:17.770 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:17.801 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:17.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:17.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:17.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:17.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:18.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:18.077 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:18.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:18.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:18.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:18.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:18.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:18.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:18.444 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:18.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:18.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:18.615 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:18.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:18.717 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:18.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:18 smithi159 ceph-mon[102519]: pgmap v1503: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 91 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 377/327960 objects degraded (0.115%); 47688/327960 objects misplaced (14.541%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:18.754 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:18.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:18 smithi042 ceph-mon[113926]: pgmap v1503: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 91 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 377/327960 objects degraded (0.115%); 47688/327960 objects misplaced (14.541%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:18.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:18 smithi042 ceph-mon[108570]: pgmap v1503: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 91 active+clean; 6.7 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 1.06k op/s; 377/327960 objects degraded (0.115%); 47688/327960 objects misplaced (14.541%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:18.855 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:18.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:18.893 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:18.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:18.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:19.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:19.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:19.131 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:19.168 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:19.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:19.270 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:19.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.304 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:19.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:19.405 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:19.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:19.544 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:19.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:19.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:19.684 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:19.722 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:19.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:19.824 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:19.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.856 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:19.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:19.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:19.981 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:19] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-09-13T15:07:19.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:19.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:20.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:20.133 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:20.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:20.235 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:20.264 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:20.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:20.366 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.402 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:20.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:20.504 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:20.536 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:20.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:20.638 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:20.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.673 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:20.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:20.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:20 smithi159 ceph-mon[102519]: pgmap v1504: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:20.774 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:20.802 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:20.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:20.817 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:20.817 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:20.817 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:20 smithi042 ceph-mon[113926]: pgmap v1504: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:20.817 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:20 smithi042 ceph-mon[108570]: pgmap v1504: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:20.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:20.941 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:20.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:21.043 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:21.077 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:21.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:21.179 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:21.211 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:21.313 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.348 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:21.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:21.449 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:21.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:21.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:21.592 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:21.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:21.721 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:21 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 570/321336 objects degraded (0.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:21.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.757 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:21.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 570/321336 objects degraded (0.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 570/321336 objects degraded (0.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:21.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:21.860 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:21.902 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:21.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:21.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:22.004 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:22.038 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:22.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:22.140 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:22.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.162 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:22.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:22.263 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:22.296 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:22.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:22.428 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:22.530 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.562 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:22.664 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:22.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.696 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:22.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:22.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:22.837 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:22.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:22 smithi042 ceph-mon[108570]: pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:07:22.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:22 smithi042 ceph-mon[113926]: pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:07:22.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:22.972 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:22.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:22.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:22.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:22 smithi159 ceph-mon[102519]: pgmap v1505: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.5 GiB data, 24 GiB used, 512 GiB / 536 GiB avail; 0 B/s wr, 628 op/s; 570/321336 objects degraded (0.177%); 45638/321336 objects misplaced (14.203%); 12 MiB/s, 187 objects/s recovering 2024-09-13T15:07:23.014 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:07:23.074 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:23.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:23.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:23.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:23.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:23.419 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:23.520 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.557 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:23.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:23.657 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:23.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:23.798 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:23.812 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:23.817 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:23.830 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:23.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:23.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:23.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:07:23.932 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:24.017 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:24.119 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:24.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:24.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:24.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:24.441 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:24.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:24.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:24.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:24.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:24.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.724 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:24.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:24.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:24 smithi042 ceph-mon[108570]: pgmap v1506: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 926 op/s; 570/310983 objects degraded (0.183%); 43661/310983 objects misplaced (14.040%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:07:24.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:24 smithi042 ceph-mon[113926]: pgmap v1506: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 926 op/s; 570/310983 objects degraded (0.183%); 43661/310983 objects misplaced (14.040%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:07:24.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:24.953 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:24.995 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:24.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:24.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:24.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:24 smithi159 ceph-mon[102519]: pgmap v1506: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 926 op/s; 570/310983 objects degraded (0.183%); 43661/310983 objects misplaced (14.040%); 16 MiB/s, 255 objects/s recovering 2024-09-13T15:07:25.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:25.119 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:25.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:25.221 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.260 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:25.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:25.362 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:25.487 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:25.523 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:25.625 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:25.657 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:25.759 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:25.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:25.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:25.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:25.930 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:25.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:25.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:26.032 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:26.069 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:26.171 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:26.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:26.313 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:26.351 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:26.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:26.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:26.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:26.586 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:26.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.623 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:26.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:26.725 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:26.761 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:26.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:26.766 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:26 smithi042 ceph-mon[113926]: pgmap v1507: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 917 op/s; 796/307644 objects degraded (0.259%); 42815/307644 objects misplaced (13.917%); 14 MiB/s, 226 objects/s recovering 2024-09-13T15:07:26.766 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 796/307644 objects degraded (0.259%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:26.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:26 smithi042 ceph-mon[108570]: pgmap v1507: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 917 op/s; 796/307644 objects degraded (0.259%); 42815/307644 objects misplaced (13.917%); 14 MiB/s, 226 objects/s recovering 2024-09-13T15:07:26.767 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 796/307644 objects degraded (0.259%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:26.863 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:26.897 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:26.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:26.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:26 smithi159 ceph-mon[102519]: pgmap v1507: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.3 GiB data, 24 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 917 op/s; 796/307644 objects degraded (0.259%); 42815/307644 objects misplaced (13.917%); 14 MiB/s, 226 objects/s recovering 2024-09-13T15:07:26.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 796/307644 objects degraded (0.259%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:27.045 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:27.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.188 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:27.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:27.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.324 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:27.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:27.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:27.456 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:27.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:27.507 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:07:27.508 INFO:teuthology.orchestra.run.smithi042.stdout: "success": "" 2024-09-13T15:07:27.508 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:07:27.558 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:27.594 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:27.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:27.695 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:27.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:27.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:27.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:27.866 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:27.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:27.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:27.968 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:28.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.006 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:28.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:28.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:28.138 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:28.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:28.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:28.273 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:28.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:28.374 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:28.414 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:28.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:28.516 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:28.554 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:28.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:28.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:28.693 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:28.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:28.795 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:28.823 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:28.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:28.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:28 smithi042 ceph-mon[108570]: pgmap v1508: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 796/301920 objects degraded (0.264%); 41725/301920 objects misplaced (13.820%); 18 MiB/s, 291 objects/s recovering 2024-09-13T15:07:28.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:28 smithi042 ceph-mon[113926]: pgmap v1508: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 796/301920 objects degraded (0.264%); 41725/301920 objects misplaced (13.820%); 18 MiB/s, 291 objects/s recovering 2024-09-13T15:07:28.926 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:28.956 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:28.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:28.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:28.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:28 smithi159 ceph-mon[102519]: pgmap v1508: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.1 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 796/301920 objects degraded (0.264%); 41725/301920 objects misplaced (13.820%); 18 MiB/s, 291 objects/s recovering 2024-09-13T15:07:29.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:29.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:29.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:29.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:29.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:29.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:29.371 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:29.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:29.473 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.503 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:29.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:29.604 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:29.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:29.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:29.729 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:29 smithi042 ceph-mon[108570]: pgmap v1509: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:29.729 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:29 smithi042 ceph-mon[113926]: pgmap v1509: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:29.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:29.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:29.883 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:29.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:29.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:29.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:29 smithi159 ceph-mon[102519]: pgmap v1509: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 13 MiB/s, 207 objects/s recovering 2024-09-13T15:07:30.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:30.063 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:30.070 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:29] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-09-13T15:07:30.165 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:30.210 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:30.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:30.311 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:30.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:30.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:30.486 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:30.536 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:07:30.536 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:07:30.588 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:30.626 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:30.728 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:30.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:30.866 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:30.901 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:30.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:30.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:30.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:30 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 999/294483 objects degraded (0.339%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:31.002 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:31.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:30 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 999/294483 objects degraded (0.339%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:31.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:30 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 999/294483 objects degraded (0.339%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:31.142 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:31.165 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:31.267 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:31.302 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:31.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:31.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:31.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.575 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:31.666 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:07:31.677 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.703 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:31.805 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:31.838 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:31.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:31.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:31.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:31.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:31 smithi159 ceph-mon[102519]: pgmap v1510: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 9.8 MiB/s, 156 objects/s recovering 2024-09-13T15:07:31.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:32.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:31 smithi042 ceph-mon[113926]: pgmap v1510: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 9.8 MiB/s, 156 objects/s recovering 2024-09-13T15:07:32.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:32.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:31 smithi042 ceph-mon[108570]: pgmap v1510: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 6.0 GiB data, 23 GiB used, 513 GiB / 536 GiB avail; 0 B/s wr, 745 op/s; 999/294483 objects degraded (0.339%); 40059/294483 objects misplaced (13.603%); 9.8 MiB/s, 156 objects/s recovering 2024-09-13T15:07:32.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:07:32.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:32.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:32.214 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:32.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:32.352 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:32.387 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:32.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:32.517 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:32.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:32.619 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:32.656 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:32.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:32.667 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=cleanup t=2024-09-13T15:07:32.29656766Z level=info msg="Completed cleanup jobs" duration=3.108013ms 2024-09-13T15:07:32.667 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=plugins.update.checker t=2024-09-13T15:07:32.476858745Z level=info msg="Update check succeeded" duration=73.76148ms 2024-09-13T15:07:32.667 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana.update.checker t=2024-09-13T15:07:32.47973191Z level=info msg="Update check succeeded" duration=76.658221ms 2024-09-13T15:07:32.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:32.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:32.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:32.901 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:32.940 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:32.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:32.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:32.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:32 smithi159 ceph-mon[102519]: osdmap e418: 8 total, 8 up, 6 in 2024-09-13T15:07:33.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:33.074 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:33.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:33.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:33.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:33.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[113926]: osdmap e418: 8 total, 8 up, 6 in 2024-09-13T15:07:33.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:33.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4183002824' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:07:33.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:32 smithi042 ceph-mon[108570]: osdmap e418: 8 total, 8 up, 6 in 2024-09-13T15:07:33.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:33.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.215 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:33.316 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:33.350 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:33.451 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:33.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:33.589 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:33.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:33.726 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:33.765 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:33.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:33 smithi042 ceph-mon[108570]: pgmap v1512: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 885 op/s; 999/284397 objects degraded (0.351%); 38089/284397 objects misplaced (13.393%); 12 MiB/s, 197 objects/s recovering 2024-09-13T15:07:33.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:33 smithi042 ceph-mon[113926]: pgmap v1512: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 885 op/s; 999/284397 objects degraded (0.351%); 38089/284397 objects misplaced (13.393%); 12 MiB/s, 197 objects/s recovering 2024-09-13T15:07:33.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:33.900 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:33.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:33.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:33.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:33 smithi159 ceph-mon[102519]: pgmap v1512: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.8 GiB data, 23 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 885 op/s; 999/284397 objects degraded (0.351%); 38089/284397 objects misplaced (13.393%); 12 MiB/s, 197 objects/s recovering 2024-09-13T15:07:34.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.039 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:34.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:34.140 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.176 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:34.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:34.277 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:34.315 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:34.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:07:34.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:34.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:34.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:34.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.594 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:34.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:34.696 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.729 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:34.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:34.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.863 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:34.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:34.873 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:34.964 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:34.998 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:34.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:34.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:35.100 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:35.139 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:35.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:35.277 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:35.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:35.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:35.410 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:35.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:35.515 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:35.550 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:35.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.691 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:35.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:35.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:35.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:35.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:35.931 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:35.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:07:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:35.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:35.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:36.075 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:36.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[108570]: pgmap v1513: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 1182/281172 objects degraded (0.420%); 37268/281172 objects misplaced (13.255%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1182/281172 objects degraded (0.420%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[113926]: pgmap v1513: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 1182/281172 objects degraded (0.420%); 37268/281172 objects misplaced (13.255%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1182/281172 objects degraded (0.420%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:36.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.102 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:36.204 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:36.241 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:35 smithi159 ceph-mon[102519]: pgmap v1513: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.7 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 882 op/s; 1182/281172 objects degraded (0.420%); 37268/281172 objects misplaced (13.255%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:35 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1182/281172 objects degraded (0.420%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:36.343 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:36.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.405 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:36.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:36.506 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:36.549 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:36.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:36.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:36.681 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:36.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:36.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:36.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:36.930 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:36.967 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:36.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:36.982 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:37.008 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:07:37.008 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:07:37.047 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:07:37.048 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:07:37.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:37.085 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:07:37.086 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:07:37.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:37.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:37.116 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:07:37.117 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:07:37.148 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:07:37.149 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:07:37.180 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:07:37.181 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:07:37.199 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:07:37.199 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:07:37.201 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:37.230 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:07:37.230 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:07:37.233 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:37.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:37.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:37.366 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:37.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.509 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:37.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:37.610 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:37.646 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:37.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:37.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:37.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:37.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:37.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:37.913 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:37.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:38.014 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:38.057 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:37 smithi042 ceph-mon[113926]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 892 op/s; 1182/275136 objects degraded (0.430%); 36315/275136 objects misplaced (13.199%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:07:38.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1301864111' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:37 smithi042 ceph-mon[108570]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 892 op/s; 1182/275136 objects degraded (0.430%); 36315/275136 objects misplaced (13.199%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:07:38.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1301864111' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:38.158 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:38.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:38.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:37 smithi159 ceph-mon[102519]: pgmap v1514: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.6 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 0 B/s wr, 892 op/s; 1182/275136 objects degraded (0.430%); 36315/275136 objects misplaced (13.199%); 12 MiB/s, 194 objects/s recovering 2024-09-13T15:07:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1301864111' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:07:38.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:38.337 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:38.438 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:38.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:38.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:38.611 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:38.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:38.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:38.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:38.855 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:38.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:38.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:38.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.035 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:39.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:39.137 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.173 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:39.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:39.274 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:39.309 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:39.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:39.411 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:39.448 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:39.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:39.549 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:39.585 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:39.687 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:39.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:39.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:39.867 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:39.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:39.899 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4] out_osds: [0, 1] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:07:39.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:07:39.900 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-09-13T15:07:39.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd in 1 2024-09-13T15:07:39.968 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:39.980 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:39 smithi042 ceph-mon[108570]: pgmap v1515: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:39.980 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:39] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:07:39.981 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:39 smithi042 ceph-mon[113926]: pgmap v1515: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:40.002 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.104 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:40.138 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:40.240 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:39 smithi159 ceph-mon[102519]: pgmap v1515: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.379 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:40.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:40.655 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:40.687 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:40.789 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:40.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:40.930 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:40.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:40.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:41.059 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:41.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:41.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:40 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1397/267315 objects degraded (0.523%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:41.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:40 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1397/267315 objects degraded (0.523%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:41.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:41.229 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:40 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1397/267315 objects degraded (0.523%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:07:41.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:41.371 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:41.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:41.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:41.508 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:41.609 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:41.642 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:41.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:41.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:41.790 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:41.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:41.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:41.934 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:41.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:41.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:42.036 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:42.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.075 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:42.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:41 smithi042 ceph-mon[108570]: pgmap v1516: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:41 smithi042 ceph-mon[113926]: pgmap v1516: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:42.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.207 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:42.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:41 smithi159 ceph-mon[102519]: pgmap v1516: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.4 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 905 op/s; 1397/267315 objects degraded (0.523%); 34586/267315 objects misplaced (12.938%); 12 MiB/s, 195 objects/s recovering 2024-09-13T15:07:42.312 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.347 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:42.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:42.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:42.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:42.587 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:42.627 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:42.729 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:42.759 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:42.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:42.898 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:42.899 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:43.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:43.029 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:43.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:43.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:43.164 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:43.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:43.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:43.293 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:07:43.304 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:43.317 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:43.406 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.451 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:43.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:43.552 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:43.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:43.699 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:43.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:43.843 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:43.858 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:07:43.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[108570]: pgmap v1517: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1397/256899 objects degraded (0.544%); 32820/256899 objects misplaced (12.775%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:07:43.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2459054397' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:43.863 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:43.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:43.887 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:43.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:43.988 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:44.037 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:44.120 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:07:44.120 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:07:44.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.120 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:164927 14.f 14:f57d1992:::benchmark_data_smithi042_319156_object40970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:44.008237+0000", 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.023376047, 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.023407777000000001, 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 164927 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.121 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008237+0000", 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008237+0000", 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008238+0000", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3099999999999996e-07 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008243+0000", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6789999999999996e-06 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008254+0000", 2024-09-13T15:07:44.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1483e-05 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008262+0000", 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4479999999999997e-06 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008366+0000", 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000103701 2024-09-13T15:07:44.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008488+0000", 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122447 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.008615+0000", 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012743499999999999 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.018221+0000", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096057790000000001 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.018507+0000", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00028570199999999998 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.018819+0000", 2024-09-13T15:07:44.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00031162300000000002 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:164936 14.f 14:fa6dd596:::benchmark_data_smithi042_319156_object40979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:44.019726+0000", 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.011887676, 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012018025999999999, 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:44.127 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 164936 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:44.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019726+0000", 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019726+0000", 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:44.129 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019727+0000", 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.297e-06 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019732+0000", 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3090000000000003e-06 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019750+0000", 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7586000000000001e-05 2024-09-13T15:07:44.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019759+0000", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8789999999999992e-06 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.019919+0000", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000160135 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.020068+0000", 2024-09-13T15:07:44.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000149171 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.020215+0000", 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014654699999999999 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.028791+0000", 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.008576162 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.029024+0000", 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00023278400000000001 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.029378+0000", 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035446400000000003 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.133 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:164939 14.9 14:9897e9f0:::benchmark_data_smithi042_319156_object40982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:44.023394+0000", 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 0.0082197659999999999, 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0084354440000000003, 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "waiting for sub ops", 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 164939 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:07:44.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023394+0000", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023394+0000", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023395+0000", 2024-09-13T15:07:44.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.28e-06 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023400+0000", 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0440000000000003e-06 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023415+0000", 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5114e-05 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.023423+0000", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.627e-06 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.028961+0000", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00553835 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.029624+0000", 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00066273600000000003 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:07:44.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:44.029748+0000", 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000123774 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 3 2024-09-13T15:07:44.138 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:07:44.138 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[113926]: pgmap v1517: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1397/256899 objects degraded (0.544%); 32820/256899 objects misplaced (12.775%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:07:44.139 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2459054397' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:44.139 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:44.139 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:44.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:43 smithi159 ceph-mon[102519]: pgmap v1517: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 1.05k op/s; 1397/256899 objects degraded (0.544%); 32820/256899 objects misplaced (12.775%); 10 MiB/s, 162 objects/s recovering 2024-09-13T15:07:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:43 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2459054397' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:07:44.278 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:44.309 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:44.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:44.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:44.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:44.550 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:44.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:44.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:44.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:44.719 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:44.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:44.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:44.860 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:44.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:44.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:44.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:44.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:44.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:45.097 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:45.131 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:45.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:45.233 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:44 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:44 smithi159 ceph-mon[102519]: osdmap e419: 8 total, 8 up, 6 in 2024-09-13T15:07:45.267 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:45.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:45.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:45.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:45.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[108570]: osdmap e419: 8 total, 8 up, 6 in 2024-09-13T15:07:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:07:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:07:45.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:44 smithi042 ceph-mon[113926]: osdmap e419: 8 total, 8 up, 6 in 2024-09-13T15:07:45.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:45.408 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:45.510 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:45.538 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:45.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:45.640 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:45.672 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:45.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:45.774 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:45.815 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:45.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:45.950 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:45.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:46.052 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:46.074 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:45 smithi042 ceph-mon[113926]: pgmap v1519: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 1653/253266 objects degraded (0.653%); 31908/253266 objects misplaced (12.599%); 11 MiB/s, 178 objects/s recovering 2024-09-13T15:07:46.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:45 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1653/253266 objects degraded (0.653%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:45 smithi042 ceph-mon[108570]: pgmap v1519: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 1653/253266 objects degraded (0.653%); 31908/253266 objects misplaced (12.599%); 11 MiB/s, 178 objects/s recovering 2024-09-13T15:07:46.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:45 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1653/253266 objects degraded (0.653%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:46.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:46.214 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:46.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:45 smithi159 ceph-mon[102519]: pgmap v1519: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.2 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 929 op/s; 1653/253266 objects degraded (0.653%); 31908/253266 objects misplaced (12.599%); 11 MiB/s, 178 objects/s recovering 2024-09-13T15:07:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:45 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1653/253266 objects degraded (0.653%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:46.316 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:46.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:46.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:46.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:46.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:46.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:46.583 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:46.619 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:46.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:46.721 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:46.762 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:46.864 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:46.904 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:46.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:47.006 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:47.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:47.176 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:47.278 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:47.308 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:47.410 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:47.443 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:47.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:47.584 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:47.686 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:47.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:47.824 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:47.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:47.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:47.962 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:47.997 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:47.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:47.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:48.098 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.122 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:48.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:48.224 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:47 smithi159 ceph-mon[102519]: pgmap v1520: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 922 op/s; 1653/247446 objects degraded (0.668%); 30954/247446 objects misplaced (12.509%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:07:48.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:48.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:47 smithi042 ceph-mon[108570]: pgmap v1520: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 922 op/s; 1653/247446 objects degraded (0.668%); 30954/247446 objects misplaced (12.509%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:07:48.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:47 smithi042 ceph-mon[113926]: pgmap v1520: 99 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 5.0 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 922 op/s; 1653/247446 objects degraded (0.668%); 30954/247446 objects misplaced (12.509%); 12 MiB/s, 186 objects/s recovering 2024-09-13T15:07:48.351 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.379 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:48.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:48.480 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.501 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:48.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:48.603 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:48.625 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:48.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:48.727 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:48.751 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:48.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:48.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:48.874 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:48.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:48.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:07:48.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:49.043 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:49.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:49.145 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:49.199 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:48 smithi159 ceph-mon[102519]: osdmap e420: 8 total, 8 up, 6 in 2024-09-13T15:07:49.301 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:49.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:49.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:48 smithi042 ceph-mon[113926]: osdmap e420: 8 total, 8 up, 6 in 2024-09-13T15:07:49.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:48 smithi042 ceph-mon[108570]: osdmap e420: 8 total, 8 up, 6 in 2024-09-13T15:07:49.439 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:49.481 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:49.583 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:49.614 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:49.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:49.716 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:49.769 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:49.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:49.872 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:49.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:49.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:50.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:50.038 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[108570]: pgmap v1522: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:50.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[108570]: osdmap e421: 8 total, 8 up, 6 in 2024-09-13T15:07:50.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:50.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:49] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:07:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[113926]: pgmap v1522: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[113926]: osdmap e421: 8 total, 8 up, 6 in 2024-09-13T15:07:50.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:49 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:50.141 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:50.152 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:49 smithi159 ceph-mon[102519]: pgmap v1522: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 1.14k op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 15 MiB/s, 242 objects/s recovering 2024-09-13T15:07:50.152 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:49 smithi159 ceph-mon[102519]: osdmap e421: 8 total, 8 up, 6 in 2024-09-13T15:07:50.152 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:49 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:07:50.186 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:50.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.325 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:50.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.469 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:50.511 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:07:50.570 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:50.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:50.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:50.711 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:50.750 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:50.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:50.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:50.894 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:50.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:50.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:50.996 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:51.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:51.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:51.067 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:50 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1920/239865 objects degraded (0.800%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:51.067 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:51.067 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:50 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1920/239865 objects degraded (0.800%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:51.067 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:51.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:51.170 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:51.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:51.224 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:51.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:50 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1920/239865 objects degraded (0.800%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:51.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:07:51.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:51.304 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:51.340 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:51.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:51.445 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:51.483 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:51.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:51.585 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.624 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:51.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:51.678 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:51 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:07:51.558+0000 7fcad2a32640 -1 mon.a@0(leader).osd e421 definitely_dead 0 2024-09-13T15:07:51.725 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:51.760 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:51.849 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:07:51.862 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:51.908 INFO:teuthology.orchestra.run.smithi042.stderr:marked in osd.1. 2024-09-13T15:07:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:51.909 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:51.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:51.932 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:51 smithi042 ceph-mon[108570]: pgmap v1524: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T15:07:51.932 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:51 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T15:07:51.932 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:51 smithi042 ceph-mon[113926]: pgmap v1524: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T15:07:51.933 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:51 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T15:07:52.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:52.034 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:52.131 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:51 smithi159 ceph-mon[102519]: pgmap v1524: 99 pgs: 2 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 92 active+clean; 4.9 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 1920/239865 objects degraded (0.800%); 29200/239865 objects misplaced (12.174%); 9.7 MiB/s, 155 objects/s recovering 2024-09-13T15:07:52.131 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:51 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-09-13T15:07:52.136 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:52.159 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:52.261 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:52.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:52.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:52.386 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:52.409 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:52.511 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:52.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:52.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:52.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:52.658 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:52.760 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:52.783 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:52.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:52.885 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:52.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:52.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:53.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:53.030 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:53.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:53.154 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T15:07:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:52 smithi159 ceph-mon[102519]: osdmap e422: 8 total, 8 up, 7 in 2024-09-13T15:07:53.256 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:53.279 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:53.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T15:07:53.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:52 smithi042 ceph-mon[108570]: osdmap e422: 8 total, 8 up, 7 in 2024-09-13T15:07:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/454922528' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-09-13T15:07:53.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:52 smithi042 ceph-mon[113926]: osdmap e422: 8 total, 8 up, 7 in 2024-09-13T15:07:53.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:53.402 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:53.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:53.503 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:53.525 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:53.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:53.626 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:53.649 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:53.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:53.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:53.774 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:53.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:53.877 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:53.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:53.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:54.001 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:54.029 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:54.131 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:54.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:53 smithi159 ceph-mon[102519]: pgmap v1526: 99 pgs: 17 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 1920/232017 objects degraded (0.828%); 14281/232017 objects misplaced (6.155%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:07:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:53 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:53 smithi159 ceph-mon[102519]: osdmap e423: 8 total, 8 up, 7 in 2024-09-13T15:07:54.276 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:54.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[113926]: pgmap v1526: 99 pgs: 17 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 1920/232017 objects degraded (0.828%); 14281/232017 objects misplaced (6.155%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:07:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:54.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[113926]: osdmap e423: 8 total, 8 up, 7 in 2024-09-13T15:07:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[108570]: pgmap v1526: 99 pgs: 17 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 853 op/s; 1920/232017 objects degraded (0.828%); 14281/232017 objects misplaced (6.155%); 12 MiB/s, 192 objects/s recovering 2024-09-13T15:07:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T15:07:54.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:53 smithi042 ceph-mon[108570]: osdmap e423: 8 total, 8 up, 7 in 2024-09-13T15:07:54.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:54.453 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:54.555 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:54.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:54.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:54.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:54.815 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:54.842 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:54.944 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:54.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:54.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:54.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:55.082 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:55.115 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:55.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:55.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:55.240 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:55.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:54 smithi159 ceph-mon[102519]: 14.2 starting backfill to osd.1 from (402'4805,419'10614] MIN to 421'10793 2024-09-13T15:07:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:54 smithi159 ceph-mon[102519]: osdmap e424: 8 total, 8 up, 7 in 2024-09-13T15:07:55.250 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:54 smithi042 ceph-mon[108570]: 14.2 starting backfill to osd.1 from (402'4805,419'10614] MIN to 421'10793 2024-09-13T15:07:55.250 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:54 smithi042 ceph-mon[108570]: osdmap e424: 8 total, 8 up, 7 in 2024-09-13T15:07:55.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:54 smithi042 ceph-mon[113926]: 14.2 starting backfill to osd.1 from (402'4805,419'10614] MIN to 421'10793 2024-09-13T15:07:55.250 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:54 smithi042 ceph-mon[113926]: osdmap e424: 8 total, 8 up, 7 in 2024-09-13T15:07:55.342 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:55.371 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:55.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:55.473 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:55.496 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:55.599 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:55.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:55.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:55.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:55.758 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:55.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:55.892 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:55.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:55.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:55.994 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:07:56.015 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:07:56.117 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:55 smithi159 ceph-mon[102519]: pgmap v1529: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 2036/230052 objects degraded (0.885%); 8843/230052 objects misplaced (3.844%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:55 smithi159 ceph-mon[102519]: osdmap e425: 8 total, 8 up, 7 in 2024-09-13T15:07:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:55 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2036/230052 objects degraded (0.885%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:55 smithi159 ceph-mon[102519]: osdmap e426: 8 total, 8 up, 7 in 2024-09-13T15:07:56.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:56.277 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[108570]: pgmap v1529: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 2036/230052 objects degraded (0.885%); 8843/230052 objects misplaced (3.844%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:56.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[108570]: osdmap e425: 8 total, 8 up, 7 in 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2036/230052 objects degraded (0.885%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[108570]: osdmap e426: 8 total, 8 up, 7 in 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[113926]: pgmap v1529: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 552 op/s; 2036/230052 objects degraded (0.885%); 8843/230052 objects misplaced (3.844%); 12 MiB/s, 199 objects/s recovering 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[113926]: osdmap e425: 8 total, 8 up, 7 in 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2036/230052 objects degraded (0.885%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:07:56.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:55 smithi042 ceph-mon[113926]: osdmap e426: 8 total, 8 up, 7 in 2024-09-13T15:07:56.380 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:56.413 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:56.515 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:56.542 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:56.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:56.633 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-09-13T15:07:56.644 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:56.674 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:56.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:56.777 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:56.801 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:56.903 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:56.929 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:56.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:57.031 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:57.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:57.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:57.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:57.200 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:57.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:57.333 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:57.435 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:07:57.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.476 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:57.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:07:57.577 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:57.620 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:57.722 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:57.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.768 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:57.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:57.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:57.906 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:57.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:58.007 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:58.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:58.141 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[108570]: pgmap v1532: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 2036/228639 objects degraded (0.890%); 8843/228639 objects misplaced (3.868%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T15:07:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.280 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.281 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[113926]: pgmap v1532: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 2036/228639 objects degraded (0.890%); 8843/228639 objects misplaced (3.868%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T15:07:58.281 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.281 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:58 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.317 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:58.418 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:58.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:58 smithi159 ceph-mon[102519]: pgmap v1532: 99 pgs: 18 peering, 2 remapped+peering, 1 unknown, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 4.7 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 281 op/s; 2036/228639 objects degraded (0.890%); 8843/228639 objects misplaced (3.868%); 2.6 MiB/s, 41 objects/s recovering 2024-09-13T15:07:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:58 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:07:58.547 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:58.569 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:58.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:58.670 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:07:58.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:07:58.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:58.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:58.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:58.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:58.941 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:58.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:59.043 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:59.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:59.167 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:59.195 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:59.297 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:07:59.329 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:59.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:07:59.397 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:07:59 smithi042 ceph-mon[113926]: osdmap e427: 8 total, 8 up, 7 in 2024-09-13T15:07:59.397 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:07:59 smithi042 ceph-mon[108570]: osdmap e427: 8 total, 8 up, 7 in 2024-09-13T15:07:59.433 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:07:59.470 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:07:59.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:07:59 smithi159 ceph-mon[102519]: osdmap e427: 8 total, 8 up, 7 in 2024-09-13T15:07:59.572 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:07:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.609 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:59.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:07:59.711 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:07:59.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:07:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:07:59.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:07:59.895 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:07:59.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:07:59.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:07:59.976 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:07:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:07:59] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-09-13T15:07:59.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:00.015 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:08:00.015 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:00.035 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:00.061 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:08:00.061 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:00.096 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:08:00.096 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:00.133 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:08:00.133 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:00.137 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:00.152 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:08:00.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:00.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:00.179 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:08:00.179 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:00.221 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:08:00.221 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:00.260 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:08:00.261 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:00.261 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: pgmap v1534: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:00.261 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-13T15:08:00.261 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: osdmap e428: 8 total, 8 up, 7 in 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2395252056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3155840937' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: pgmap v1534: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: osdmap e428: 8 total, 8 up, 7 in 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2395252056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3155840937' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.262 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:00 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:00.299 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:08:00.300 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.314 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:00.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:00.333 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:00.333 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:00.416 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:00.454 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:00.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: pgmap v1534: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: osdmap e428: 8 total, 8 up, 7 in 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2395252056' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3155840937' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:00 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:08:00.556 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:00.586 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:00.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:00.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:00.726 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:00.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:00.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:00.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:00.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:00.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:00.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:01.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:01.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:01.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:01 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:01 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:01 smithi159 ceph-mon[102519]: osdmap e429: 8 total, 8 up, 7 in 2024-09-13T15:08:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:01 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2037/223101 objects degraded (0.913%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:01.274 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:01.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[108570]: osdmap e429: 8 total, 8 up, 7 in 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2037/223101 objects degraded (0.913%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[113926]: osdmap e429: 8 total, 8 up, 7 in 2024-09-13T15:08:01.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:01 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2037/223101 objects degraded (0.913%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:01.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:01.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.418 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:01.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:01.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.564 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:01.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:01.634 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:08:01.635 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:01.635 INFO:tasks.thrashosds.thrasher:Reweighting osd 1 to 0.4708458166358874 2024-09-13T15:08:01.635 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 0.4708458166358874 2024-09-13T15:08:01.666 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:01.713 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:01.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:01.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:01.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:01.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:01.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:01.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:08:01.997 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:01.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:01.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:02.099 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:02.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:02.229 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:02.261 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:02.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:02 smithi042 ceph-mon[113926]: pgmap v1537: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:02.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:02 smithi042 ceph-mon[108570]: pgmap v1537: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:02.363 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.406 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:02.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:02.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:02 smithi159 ceph-mon[102519]: pgmap v1537: 99 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 4.5 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 2037/223101 objects degraded (0.913%); 9234/223101 objects misplaced (4.139%); 3.1 MiB/s, 50 objects/s recovering 2024-09-13T15:08:02.508 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:02.544 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:02.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:02.646 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:02.681 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:02.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:02.756 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:02.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:02.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:02.914 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:02.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:02.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:02.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:03.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:03.084 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:03.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:03.107 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:08:03.187 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:03.217 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:03.319 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:03.366 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:03.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.367 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:03.468 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:03.495 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:03.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:03.597 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:03.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:03.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:03.806 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:03.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:03.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:03.953 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:03.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:04.056 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:04.091 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:04.183 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:04.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:04.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:04.312 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:04 smithi042 ceph-mon[108570]: pgmap v1538: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 2037/213999 objects degraded (0.952%); 8495/213999 objects misplaced (3.970%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:04.313 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:04 smithi042 ceph-mon[113926]: pgmap v1538: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 2037/213999 objects degraded (0.952%); 8495/213999 objects misplaced (3.970%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:04.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.383 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:04.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:04.483 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.486 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:08:04.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:04 smithi159 ceph-mon[102519]: pgmap v1538: 99 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 2037/213999 objects degraded (0.952%); 8495/213999 objects misplaced (3.970%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:04.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:04.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:04.616 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:04.646 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:04.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:04.748 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:04.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:04.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:04.880 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:04.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:04.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:04.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:05.020 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:05.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:05.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.189 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:05.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:05.291 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:05.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:05.409 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:05.419 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:05.442 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:05.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:05.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:05.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:05.688 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:05.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:05.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:05.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:05.882 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:05.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:05.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:05.984 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:06.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:06.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:06.118 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.158 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:06.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.261 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.305 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:06.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.407 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.445 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:06.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:06 smithi159 ceph-mon[102519]: pgmap v1539: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:06 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2160/210795 objects degraded (1.025%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:06.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:06.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.561 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[108570]: pgmap v1539: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:06.561 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2160/210795 objects degraded (1.025%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:06.561 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:06.561 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[113926]: pgmap v1539: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 603 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 2.5 MiB/s, 40 objects/s recovering 2024-09-13T15:08:06.561 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2160/210795 objects degraded (1.025%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:06.562 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:06.587 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:06.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.689 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:06.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:06.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:06.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:06.853 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:06.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:06.955 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:06.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:06.988 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:06.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:07.089 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:07.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.129 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:07.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:07.230 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:07.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:07.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:07.370 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.404 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:07.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:07.505 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.538 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:07.640 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.682 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:07.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:07.783 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:07.824 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:07.848 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:07.926 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:07.961 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:07.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:07.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.063 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:08.109 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:08.180 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 0.470846 (7889) 2024-09-13T15:08:08.212 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.240 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:08.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.341 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:08.366 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:08.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:08.468 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:08.491 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:08 smithi159 ceph-mon[102519]: pgmap v1540: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 864 KiB/s, 13 objects/s recovering 2024-09-13T15:08:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1620891736' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:08 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:08.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[113926]: pgmap v1540: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 864 KiB/s, 13 objects/s recovering 2024-09-13T15:08:08.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1620891736' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[108570]: pgmap v1540: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.3 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 512 op/s; 2160/210795 objects degraded (1.025%); 7997/210795 objects misplaced (3.794%); 864 KiB/s, 13 objects/s recovering 2024-09-13T15:08:08.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1620891736' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.600 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:08 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874} : dispatch 2024-09-13T15:08:08.617 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:08.719 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:08.742 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:08.844 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.866 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:08.967 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:08.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:08.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:09.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:09.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:09.215 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:09.237 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:09.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874}]': finished 2024-09-13T15:08:09.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:09 smithi042 ceph-mon[113926]: osdmap e430: 8 total, 8 up, 7 in 2024-09-13T15:08:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874}]': finished 2024-09-13T15:08:09.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:09 smithi042 ceph-mon[108570]: osdmap e430: 8 total, 8 up, 7 in 2024-09-13T15:08:09.362 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:09.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:09.464 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:09.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 0.4708458166358874}]': finished 2024-09-13T15:08:09.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:09 smithi159 ceph-mon[102519]: osdmap e430: 8 total, 8 up, 7 in 2024-09-13T15:08:09.587 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:09.608 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:09.710 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.731 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:09.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:09.855 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:09.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:09.957 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:09.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:09.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:09.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:10.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:10.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:09] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-09-13T15:08:10.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:10.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:10.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:10.230 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:10.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:10.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:10.355 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:10.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:10.478 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:10.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:10 smithi159 ceph-mon[102519]: pgmap v1542: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:08:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:10 smithi159 ceph-mon[102519]: osdmap e431: 8 total, 8 up, 7 in 2024-09-13T15:08:10.581 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:10.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:10 smithi042 ceph-mon[113926]: pgmap v1542: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:08:10.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:10 smithi042 ceph-mon[113926]: osdmap e431: 8 total, 8 up, 7 in 2024-09-13T15:08:10.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:10 smithi042 ceph-mon[108570]: pgmap v1542: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 1.3 MiB/s, 20 objects/s recovering 2024-09-13T15:08:10.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:10 smithi042 ceph-mon[108570]: osdmap e431: 8 total, 8 up, 7 in 2024-09-13T15:08:10.612 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:10.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:10.714 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:10.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:10.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:10.883 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:10.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:10.985 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:11.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:11.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:11.125 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:11.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.165 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:11.266 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:11.295 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:11.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:11.435 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:11.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:11 smithi159 ceph-mon[102519]: osdmap e432: 8 total, 8 up, 7 in 2024-09-13T15:08:11.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:11 smithi159 ceph-mon[102519]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 429'11557 2024-09-13T15:08:11.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:11 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2401/196308 objects degraded (1.223%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:11.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[113926]: osdmap e432: 8 total, 8 up, 7 in 2024-09-13T15:08:11.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[113926]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 429'11557 2024-09-13T15:08:11.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2401/196308 objects degraded (1.223%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:11.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[108570]: osdmap e432: 8 total, 8 up, 7 in 2024-09-13T15:08:11.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[108570]: 14.2 starting backfill to osd.5 from (0'0,0'0] MAX to 429'11557 2024-09-13T15:08:11.534 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:11 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2401/196308 objects degraded (1.223%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:11.537 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:11.559 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:11.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:11.698 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:11.800 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:11.834 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:11.936 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:11.964 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:11.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:12.066 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:12.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:12.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:12.207 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:12.241 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:12.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:12.381 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:12.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:12 smithi159 ceph-mon[102519]: pgmap v1545: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 746 KiB/s, 11 objects/s recovering 2024-09-13T15:08:12.509 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:12.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:12 smithi042 ceph-mon[113926]: pgmap v1545: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 746 KiB/s, 11 objects/s recovering 2024-09-13T15:08:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:12 smithi042 ceph-mon[108570]: pgmap v1545: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 96 active+clean; 4.0 GiB data, 21 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 2401/196308 objects degraded (1.223%); 7393/196308 objects misplaced (3.766%); 746 KiB/s, 11 objects/s recovering 2024-09-13T15:08:12.611 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:12.648 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:12.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:12.750 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:12.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:12.883 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:12.917 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:12.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:13.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:13.041 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:13.143 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:13.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:13.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:13.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:13.311 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:13.413 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:13.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:13.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:13.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:13.592 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:13.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:13.694 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.727 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:13.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:13.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:13.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:13.864 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:13.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:13.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:14.002 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:14.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:14.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:14.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:14.268 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:14.370 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:14.406 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:14.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:14.469 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:14 smithi042 ceph-mon[108570]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 2401/190365 objects degraded (1.261%); 11321/190365 objects misplaced (5.947%); 747 KiB/s, 11 objects/s recovering 2024-09-13T15:08:14.470 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:14 smithi042 ceph-mon[113926]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 2401/190365 objects degraded (1.261%); 11321/190365 objects misplaced (5.947%); 747 KiB/s, 11 objects/s recovering 2024-09-13T15:08:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:14 smithi159 ceph-mon[102519]: pgmap v1546: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 1.13k op/s; 2401/190365 objects degraded (1.261%); 11321/190365 objects misplaced (5.947%); 747 KiB/s, 11 objects/s recovering 2024-09-13T15:08:14.508 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:14.545 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:14.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:14.684 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:14.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:14.827 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:14.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:14.929 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:14.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:14.963 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:14.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:15.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:15.094 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:15.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:15.196 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:15.238 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:15.340 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.374 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:15.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:15.455 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:15.455 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:15.475 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:15.514 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:15.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:15.616 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:15.651 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:15.753 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:15.787 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:15.889 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:15.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:15.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:15.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:16.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:16.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:16.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:16.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:16.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.208 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:16.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:16.309 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:08:16.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:16.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:16.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:16.443 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:16.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:16 smithi159 ceph-mon[102519]: pgmap v1547: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:08:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:16 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2402/188520 objects degraded (1.274%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:16.554 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:16 smithi042 ceph-mon[113926]: pgmap v1547: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:08:16.554 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:16 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2402/188520 objects degraded (1.274%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:16.554 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:16 smithi042 ceph-mon[108570]: pgmap v1547: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 2.8 MiB/s, 45 objects/s recovering 2024-09-13T15:08:16.554 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:16 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2402/188520 objects degraded (1.274%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:16.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:16.613 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:16.645 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:16.715 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:16.748 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:16.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:16.850 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:16.885 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:16.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:16.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:16.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.019 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:17.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:17.121 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:17.153 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:17.255 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:17.290 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:17.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:17.392 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:17.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:17.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:17.533 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:17.571 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:17.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:17.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:17.704 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:17.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:17.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:17.846 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:17.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:17.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:17.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:18.007 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:18.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:18.109 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.141 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:18.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:18.242 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:18.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:18.385 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:18.410 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:18 smithi159 ceph-mon[102519]: pgmap v1548: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:18.417 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:18.519 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:18.551 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:18 smithi042 ceph-mon[113926]: pgmap v1548: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:18.551 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:18 smithi042 ceph-mon[108570]: pgmap v1548: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.8 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 0 B/s wr, 324 op/s; 2402/188520 objects degraded (1.274%); 10930/188520 objects misplaced (5.798%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:18.560 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:18.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:18.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:18.694 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:18.744 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:18.744 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:18.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:18.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:18.931 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:18.974 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:18.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:19.075 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:19.113 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:19.145 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:08:19.145 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:19.184 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:08:19.184 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:19.217 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:19.227 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:08:19.228 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:19.255 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:19.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:19.273 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:08:19.273 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:19.322 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:08:19.323 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:19.355 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:08:19.355 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:19.357 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:19.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:19.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:19.391 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:08:19.391 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:19.391 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2024-09-13T15:08:19.391 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:08:19.392 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:08:19.393 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:19.426 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:08:19.427 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:19.445 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3249453193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:19.445 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3249453193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:19.488 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3249453193' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:19.534 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:08:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.550 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:19.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:19.651 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:19.701 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:19.803 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:19.864 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:19.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:19.896 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:19.965 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:20.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.003 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:20.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:20.100 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:19] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.51.0" 2024-09-13T15:08:20.103 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:20.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.155 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:20.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:20.224 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:08:20.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:20.282 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:20.368 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:20 smithi159 ceph-mon[102519]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:08:20.368 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:20.384 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:20.423 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:20.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:20.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:20.561 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:20.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:20.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:20 smithi042 ceph-mon[113926]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:08:20.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:20.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:20 smithi042 ceph-mon[108570]: pgmap v1549: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 783 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 2.0 MiB/s, 32 objects/s recovering 2024-09-13T15:08:20.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:20.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.703 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:20.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:20.804 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:20.844 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:20.946 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:20.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:20.982 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:20.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:21.083 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:21.119 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:21.220 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:21.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.274 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:21.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:21.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: osdmap e433: 8 total, 8 up, 7 in 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2554/173763 objects degraded (1.470%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: osdmap e433: 8 total, 8 up, 7 in 2024-09-13T15:08:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2554/173763 objects degraded (1.470%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:21.350 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:21.375 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:21.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:21.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:21.477 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:08:21.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4068345281' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:21.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: osdmap e433: 8 total, 8 up, 7 in 2024-09-13T15:08:21.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2554/173763 objects degraded (1.470%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:21.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:21.478 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:21.529 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:21.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:21.653 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:21.668 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:21.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:21.824 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:21.868 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:21.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:21.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:21.970 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.017 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:22.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:22.107 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:08:22.107 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:08:22.107 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:08:22.108 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:08:22.108 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:08:22.118 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:22.160 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:22.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:22.262 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:22.293 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:22.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:22.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:22.501 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:22 smithi159 ceph-mon[102519]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:22.528 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:22.555 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:22 smithi042 ceph-mon[108570]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:22.555 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:22 smithi042 ceph-mon[113926]: pgmap v1551: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.5 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 751 op/s; 2554/173763 objects degraded (1.470%); 10079/173763 objects misplaced (5.800%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:22.562 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:22.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:22.664 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.702 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:22.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:22.803 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:22.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:22.861 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:08:22.938 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:22.994 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:22.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:23.096 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:23.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:08:23.137 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:23.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:23.228 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:23.241 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:23.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.301 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:23.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:23.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:23.440 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:23.542 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:23.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.589 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:23.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:23.593 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:08:23.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:23.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.727 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:23.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:23.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:23.859 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:23.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:23.961 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:23.998 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:23.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:23.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:24.100 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:24.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.143 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:24.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:24.244 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:24.290 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:24.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:24.392 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.433 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:24.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:24 smithi159 ceph-mon[102519]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 2554/163098 objects degraded (1.566%); 9805/163098 objects misplaced (6.012%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:24.535 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:24 smithi042 ceph-mon[108570]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 2554/163098 objects degraded (1.566%); 9805/163098 objects misplaced (6.012%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:24.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:24.536 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:24 smithi042 ceph-mon[113926]: pgmap v1552: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 2554/163098 objects degraded (1.566%); 9805/163098 objects misplaced (6.012%); 1.9 MiB/s, 30 objects/s recovering 2024-09-13T15:08:24.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:24.665 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:24.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:24.756 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:24.774 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:08:24.776 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:08:24.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:24.858 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:24.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:24.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:24.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:25.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:25.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.136 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:25.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:25.237 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:25.269 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:25.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:25.371 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:25.402 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:25.504 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:25.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:25.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:25.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.682 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:25.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:25.783 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:25.815 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:25.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:25.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:25.958 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:25.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:25.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:26.060 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:26.120 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1552,"stamp":"2024-09-13T15:08:22.774093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3561756669,"num_objects":54366,"num_object_clones":0,"num_object_copies":163098,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":2554,"num_objects_misplaced":9805,"num_objects_unfound":0,"num_objects_dirty":54366,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193634,"num_write_kb":7933090,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16397,"num_bytes_recovered":1069946544,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104559,"ondisk_log_size":104559,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":313,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":20321604,"kb_used_data":11156352,"kb_used_omap":22,"kb_used_meta":9164905,"kb_avail":635894460,"statfs":{"total":671965249536,"available":651155927040,"internally_reserved":0,"allocated":11424104448,"data_stored":11407842520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":9384862951},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,3,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-595656704,"num_objects":-9089,"num_object_clones":0,"num_object_copies":-27267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":153,"num_objects_misplaced":-1516,"num_objects_unfound":0,"num_objects_dirty":-9089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9089,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":20250624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002756"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":701,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.244126+0000","last_change":"2024-09-13T15:06:40.609190+0000","last_active":"2024-09-13T15:08:20.244126+0000","last_peered":"2024-09-13T15:08:20.244126+0000","last_clean":"2024-09-13T15:08:20.244126+0000","last_became_active":"2024-09-13T15:06:40.608791+0000","last_became_peered":"2024-09-13T15:06:40.608791+0000","last_unstale":"2024-09-13T15:08:20.244126+0000","last_undegraded":"2024-09-13T15:08:20.244126+0000","last_fullsized":"2024-09-13T15:08:20.244126+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:19.550059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":127,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.593289+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:08:10.593289+0000","last_peered":"2024-09-13T15:08:10.593289+0000","last_clean":"2024-09-13T15:08:10.593289+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:08:10.593289+0000","last_undegraded":"2024-09-13T15:08:10.593289+0000","last_fullsized":"2024-09-13T15:08:10.593289+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":258,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.252841+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:08:20.252841+0000","last_peered":"2024-09-13T15:08:20.252841+0000","last_clean":"2024-09-13T15:08:20.252841+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:08:20.252841+0000","last_undegraded":"2024-09-13T15:08:20.252841+0000","last_fullsized":"2024-09-13T15:08:20.252841+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":470,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.249493+0000","last_change":"2024-09-13T15:06:40.607992+0000","last_active":"2024-09-13T15:08:20.249493+0000","last_peered":"2024-09-13T15:08:20.249493+0000","last_clean":"2024-09-13T15:08:20.249493+0000","last_became_active":"2024-09-13T15:06:40.607427+0000","last_became_peered":"2024-09-13T15:06:40.607427+0000","last_unstale":"2024-09-13T15:08:20.249493+0000","last_undegraded":"2024-09-13T15:08:20.249493+0000","last_fullsized":"2024-09-13T15:08:20.249493+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:57.514127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":464,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.248750+0000","last_change":"2024-09-13T15:03:44.128445+0000","last_active":"2024-09-13T15:08:20.248750+0000","last_peered":"2024-09-13T15:08:20.248750+0000","last_clean":"2024-09-13T15:08:20.248750+0000","last_became_active":"2024-09-13T15:03:44.128047+0000","last_became_peered":"2024-09-13T15:03:44.128047+0000","last_unstale":"2024-09-13T15:08:20.248750+0000","last_undegraded":"2024-09-13T15:08:20.248750+0000","last_fullsized":"2024-09-13T15:08:20.248750+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:22.359631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":121,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592384+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:08:10.592384+0000","last_peered":"2024-09-13T15:08:10.592384+0000","last_clean":"2024-09-13T15:08:10.592384+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:08:10.592384+0000","last_undegraded":"2024-09-13T15:08:10.592384+0000","last_fullsized":"2024-09-13T15:08:10.592384+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":727,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.245200+0000","last_change":"2024-09-13T15:07:52.945830+0000","last_active":"2024-09-13T15:08:20.245200+0000","last_peered":"2024-09-13T15:08:20.245200+0000","last_clean":"2024-09-13T15:08:20.245200+0000","last_became_active":"2024-09-13T15:07:52.945465+0000","last_became_peered":"2024-09-13T15:07:52.945465+0000","last_unstale":"2024-09-13T15:08:20.245200+0000","last_undegraded":"2024-09-13T15:08:20.245200+0000","last_fullsized":"2024-09-13T15:08:20.245200+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:15.475946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":527,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179642+0000","last_change":"2024-09-13T15:03:44.130129+0000","last_active":"2024-09-13T15:08:10.179642+0000","last_peered":"2024-09-13T15:08:10.179642+0000","last_clean":"2024-09-13T15:08:10.179642+0000","last_became_active":"2024-09-13T15:03:44.114883+0000","last_became_peered":"2024-09-13T15:03:44.114883+0000","last_unstale":"2024-09-13T15:08:10.179642+0000","last_undegraded":"2024-09-13T15:08:10.179642+0000","last_fullsized":"2024-09-13T15:08:10.179642+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:54.544220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"433'12223","reported_seq":24393,"reported_epoch":433,"state":"active+recovery_wait+remapped","last_fresh":"2024-09-13T15:08:21.870860+0000","last_change":"2024-09-13T15:08:10.590344+0000","last_active":"2024-09-13T15:08:21.870860+0000","last_peered":"2024-09-13T15:08:21.870860+0000","last_clean":"2024-09-13T15:08:08.163445+0000","last_became_active":"2024-09-13T15:08:10.551533+0000","last_became_peered":"2024-09-13T15:08:10.551533+0000","last_unstale":"2024-09-13T15:08:21.870860+0000","last_undegraded":"2024-09-13T15:08:21.870860+0000","last_fullsized":"2024-09-13T15:08:21.870860+0000","mapping_epoch":431,"log_start":"403'5605","ondisk_log_start":"403'5605","created":395,"last_epoch_clean":428,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6618,"log_dups_size":0,"ondisk_log_size":6618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:54:21.159648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742208,"num_objects":3353,"num_object_clones":0,"num_object_copies":10059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3353,"num_objects_unfound":0,"num_objects_dirty":3353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12223,"num_write_kb":498432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6036,"num_bytes_recovered":395509760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":3353}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":163,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:16.796278+0000","last_change":"2024-09-13T15:06:40.604087+0000","last_active":"2024-09-13T15:08:16.796278+0000","last_peered":"2024-09-13T15:08:16.796278+0000","last_clean":"2024-09-13T15:08:16.796278+0000","last_became_active":"2024-09-13T15:06:40.600583+0000","last_became_peered":"2024-09-13T15:06:40.600583+0000","last_unstale":"2024-09-13T15:08:16.796278+0000","last_undegraded":"2024-09-13T15:08:16.796278+0000","last_fullsized":"2024-09-13T15:08:16.796278+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:09.743062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":93,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179348+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:08:10.179348+0000","last_peered":"2024-09-13T15:08:10.179348+0000","last_clean":"2024-09-13T15:08:10.179348+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:08:10.179348+0000","last_undegraded":"2024-09-13T15:08:10.179348+0000","last_fullsized":"2024-09-13T15:08:10.179348+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"433'12391","reported_seq":27533,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:22.646627+0000","last_change":"2024-09-13T15:07:16.487233+0000","last_active":"2024-09-13T15:08:22.646627+0000","last_peered":"2024-09-13T15:08:22.646627+0000","last_clean":"2024-09-13T15:08:22.646627+0000","last_became_active":"2024-09-13T15:07:16.476951+0000","last_became_peered":"2024-09-13T15:07:16.476951+0000","last_unstale":"2024-09-13T15:08:22.646627+0000","last_undegraded":"2024-09-13T15:08:22.646627+0000","last_fullsized":"2024-09-13T15:08:22.646627+0000","mapping_epoch":416,"log_start":"405'6926","ondisk_log_start":"405'6926","created":395,"last_epoch_clean":417,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5465,"log_dups_size":0,"ondisk_log_size":5465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:39:26.697060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674368,"num_objects":3413,"num_object_clones":0,"num_object_copies":10239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12391,"num_write_kb":505728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7512,"num_bytes_recovered":492306432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":152,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:16.796249+0000","last_change":"2024-09-13T15:07:52.949239+0000","last_active":"2024-09-13T15:08:16.796249+0000","last_peered":"2024-09-13T15:08:16.796249+0000","last_clean":"2024-09-13T15:08:16.796249+0000","last_became_active":"2024-09-13T15:07:52.934781+0000","last_became_peered":"2024-09-13T15:07:52.934781+0000","last_unstale":"2024-09-13T15:08:16.796249+0000","last_undegraded":"2024-09-13T15:08:16.796249+0000","last_fullsized":"2024-09-13T15:08:16.796249+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:19.037785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":149,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592661+0000","last_change":"2024-09-13T15:08:09.184474+0000","last_active":"2024-09-13T15:08:10.592661+0000","last_peered":"2024-09-13T15:08:10.592661+0000","last_clean":"2024-09-13T15:08:10.592661+0000","last_became_active":"2024-09-13T15:08:09.184102+0000","last_became_peered":"2024-09-13T15:08:09.184102+0000","last_unstale":"2024-09-13T15:08:10.592661+0000","last_undegraded":"2024-09-13T15:08:10.592661+0000","last_fullsized":"2024-09-13T15:08:10.592661+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:33.422950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"432'11700","reported_seq":11787,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:18.299029+0000","last_change":"2024-09-13T15:05:11.537086+0000","last_active":"2024-09-13T15:08:18.299029+0000","last_peered":"2024-09-13T15:08:18.299029+0000","last_clean":"2024-09-13T15:08:18.299029+0000","last_became_active":"2024-09-13T15:05:11.536730+0000","last_became_peered":"2024-09-13T15:05:11.536730+0000","last_unstale":"2024-09-13T15:08:18.299029+0000","last_undegraded":"2024-09-13T15:08:18.299029+0000","last_fullsized":"2024-09-13T15:08:18.299029+0000","mapping_epoch":395,"log_start":"402'4548","ondisk_log_start":"402'4548","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7152,"log_dups_size":0,"ondisk_log_size":7152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:01:15.409500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233963520,"num_objects":3570,"num_object_clones":0,"num_object_copies":10710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11700,"num_write_kb":488640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":105,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.233205+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:08:20.233205+0000","last_peered":"2024-09-13T15:08:20.233205+0000","last_clean":"2024-09-13T15:08:20.233205+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:08:20.233205+0000","last_undegraded":"2024-09-13T15:08:20.233205+0000","last_fullsized":"2024-09-13T15:08:20.233205+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":427,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179547+0000","last_change":"2024-09-13T15:07:52.932503+0000","last_active":"2024-09-13T15:08:10.179547+0000","last_peered":"2024-09-13T15:08:10.179547+0000","last_clean":"2024-09-13T15:08:10.179547+0000","last_became_active":"2024-09-13T15:07:52.932202+0000","last_became_peered":"2024-09-13T15:07:52.932202+0000","last_unstale":"2024-09-13T15:08:10.179547+0000","last_undegraded":"2024-09-13T15:08:10.179547+0000","last_fullsized":"2024-09-13T15:08:10.179547+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:59.204152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"433'12380","reported_seq":12478,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:22.644403+0000","last_change":"2024-09-13T15:07:53.957792+0000","last_active":"2024-09-13T15:08:22.644403+0000","last_peered":"2024-09-13T15:08:22.644403+0000","last_clean":"2024-09-13T15:08:22.644403+0000","last_became_active":"2024-09-13T15:07:53.925947+0000","last_became_peered":"2024-09-13T15:07:53.925947+0000","last_unstale":"2024-09-13T15:08:22.644403+0000","last_undegraded":"2024-09-13T15:08:22.644403+0000","last_fullsized":"2024-09-13T15:08:22.644403+0000","mapping_epoch":423,"log_start":"405'6912","ondisk_log_start":"405'6912","created":395,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5468,"log_dups_size":0,"ondisk_log_size":5468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:49:44.679368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211550208,"num_objects":3228,"num_object_clones":0,"num_object_copies":9684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12380,"num_write_kb":499456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":260,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.244300+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:08:20.244300+0000","last_peered":"2024-09-13T15:08:20.244300+0000","last_clean":"2024-09-13T15:08:20.244300+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:08:20.244300+0000","last_undegraded":"2024-09-13T15:08:20.244300+0000","last_fullsized":"2024-09-13T15:08:20.244300+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.235653+0000","last_change":"2024-09-13T15:07:52.937803+0000","last_active":"2024-09-13T15:08:20.235653+0000","last_peered":"2024-09-13T15:08:20.235653+0000","last_clean":"2024-09-13T15:08:20.235653+0000","last_became_active":"2024-09-13T15:07:52.935958+0000","last_became_peered":"2024-09-13T15:07:52.935958+0000","last_unstale":"2024-09-13T15:08:20.235653+0000","last_undegraded":"2024-09-13T15:08:20.235653+0000","last_fullsized":"2024-09-13T15:08:20.235653+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:15.989300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"432'12152","reported_seq":15746,"reported_epoch":432,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T15:08:18.321335+0000","last_change":"2024-09-13T15:07:04.974305+0000","last_active":"2024-09-13T15:08:18.321335+0000","last_peered":"2024-09-13T15:08:18.321335+0000","last_clean":"2024-09-13T15:06:40.113598+0000","last_became_active":"2024-09-13T15:06:42.000868+0000","last_became_peered":"2024-09-13T15:06:42.000868+0000","last_unstale":"2024-09-13T15:08:18.321335+0000","last_undegraded":"2024-09-13T15:06:41.651372+0000","last_fullsized":"2024-09-13T15:06:41.602373+0000","mapping_epoch":407,"log_start":"402'4954","ondisk_log_start":"402'4954","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7198,"log_dups_size":0,"ondisk_log_size":7198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232390656,"num_objects":3546,"num_object_clones":0,"num_object_copies":10638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2553,"num_objects_misplaced":2899,"num_objects_unfound":0,"num_objects_dirty":3546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12152,"num_write_kb":502336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1751,"num_bytes_recovered":114753536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":2899},{"shards":"2,3","objects":-1906}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":1046,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.252536+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:08:20.252536+0000","last_peered":"2024-09-13T15:08:20.252536+0000","last_clean":"2024-09-13T15:08:20.252536+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:08:20.252536+0000","last_undegraded":"2024-09-13T15:08:20.252536+0000","last_fullsized":"2024-09-13T15:08:20.252536+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":780,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.245315+0000","last_change":"2024-09-13T15:06:40.613530+0000","last_active":"2024-09-13T15:08:20.245315+0000","last_peered":"2024-09-13T15:08:20.245315+0000","last_clean":"2024-09-13T15:08:20.245315+0000","last_became_active":"2024-09-13T15:06:40.608937+0000","last_became_peered":"2024-09-13T15:06:40.608937+0000","last_unstale":"2024-09-13T15:08:20.245315+0000","last_undegraded":"2024-09-13T15:08:20.245315+0000","last_fullsized":"2024-09-13T15:08:20.245315+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:49.728732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:16.796195+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:08:16.796195+0000","last_peered":"2024-09-13T15:08:16.796195+0000","last_clean":"2024-09-13T15:08:16.796195+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:08:16.796195+0000","last_undegraded":"2024-09-13T15:08:16.796195+0000","last_fullsized":"2024-09-13T15:08:16.796195+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"433'12106","reported_seq":12154,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.775911+0000","last_change":"2024-09-13T15:05:11.537076+0000","last_active":"2024-09-13T15:08:21.775911+0000","last_peered":"2024-09-13T15:08:21.775911+0000","last_clean":"2024-09-13T15:08:21.775911+0000","last_became_active":"2024-09-13T15:05:11.536775+0000","last_became_peered":"2024-09-13T15:05:11.536775+0000","last_unstale":"2024-09-13T15:08:21.775911+0000","last_undegraded":"2024-09-13T15:08:21.775911+0000","last_fullsized":"2024-09-13T15:08:21.775911+0000","mapping_epoch":395,"log_start":"404'5648","ondisk_log_start":"404'5648","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:09:38.213878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222035968,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12106,"num_write_kb":495808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":542,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180227+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:08:10.180227+0000","last_peered":"2024-09-13T15:08:10.180227+0000","last_clean":"2024-09-13T15:08:10.180227+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:08:10.180227+0000","last_undegraded":"2024-09-13T15:08:10.180227+0000","last_fullsized":"2024-09-13T15:08:10.180227+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":466,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179699+0000","last_change":"2024-09-13T15:06:40.605335+0000","last_active":"2024-09-13T15:08:10.179699+0000","last_peered":"2024-09-13T15:08:10.179699+0000","last_clean":"2024-09-13T15:08:10.179699+0000","last_became_active":"2024-09-13T15:06:40.604777+0000","last_became_peered":"2024-09-13T15:06:40.604777+0000","last_unstale":"2024-09-13T15:08:10.179699+0000","last_undegraded":"2024-09-13T15:08:10.179699+0000","last_fullsized":"2024-09-13T15:08:10.179699+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:26.565387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:16.796216+0000","last_change":"2024-09-13T15:07:52.933247+0000","last_active":"2024-09-13T15:08:16.796216+0000","last_peered":"2024-09-13T15:08:16.796216+0000","last_clean":"2024-09-13T15:08:16.796216+0000","last_became_active":"2024-09-13T15:07:52.932736+0000","last_became_peered":"2024-09-13T15:07:52.932736+0000","last_unstale":"2024-09-13T15:08:16.796216+0000","last_undegraded":"2024-09-13T15:08:16.796216+0000","last_fullsized":"2024-09-13T15:08:16.796216+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:35.037964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"433'11919","reported_seq":11966,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.921372+0000","last_change":"2024-09-13T15:05:11.105322+0000","last_active":"2024-09-13T15:08:21.921372+0000","last_peered":"2024-09-13T15:08:21.921372+0000","last_clean":"2024-09-13T15:08:21.921372+0000","last_became_active":"2024-09-13T15:05:11.104691+0000","last_became_peered":"2024-09-13T15:05:11.104691+0000","last_unstale":"2024-09-13T15:08:21.921372+0000","last_undegraded":"2024-09-13T15:08:21.921372+0000","last_fullsized":"2024-09-13T15:08:21.921372+0000","mapping_epoch":395,"log_start":"403'5361","ondisk_log_start":"403'5361","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6558,"log_dups_size":0,"ondisk_log_size":6558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:38:23.376418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222101504,"num_objects":3389,"num_object_clones":0,"num_object_copies":10167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11919,"num_write_kb":489856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592456+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:08:10.592456+0000","last_peered":"2024-09-13T15:08:10.592456+0000","last_clean":"2024-09-13T15:08:10.592456+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:08:10.592456+0000","last_undegraded":"2024-09-13T15:08:10.592456+0000","last_fullsized":"2024-09-13T15:08:10.592456+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":131,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:16.796258+0000","last_change":"2024-09-13T15:06:40.605582+0000","last_active":"2024-09-13T15:08:16.796258+0000","last_peered":"2024-09-13T15:08:16.796258+0000","last_clean":"2024-09-13T15:08:16.796258+0000","last_became_active":"2024-09-13T15:06:40.605161+0000","last_became_peered":"2024-09-13T15:06:40.605161+0000","last_unstale":"2024-09-13T15:08:16.796258+0000","last_undegraded":"2024-09-13T15:08:16.796258+0000","last_fullsized":"2024-09-13T15:08:16.796258+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:32.977866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"433'12164","reported_seq":12212,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.767476+0000","last_change":"2024-09-13T15:05:11.103992+0000","last_active":"2024-09-13T15:08:21.767476+0000","last_peered":"2024-09-13T15:08:21.767476+0000","last_clean":"2024-09-13T15:08:21.767476+0000","last_became_active":"2024-09-13T15:05:11.103694+0000","last_became_peered":"2024-09-13T15:05:11.103694+0000","last_unstale":"2024-09-13T15:08:21.767476+0000","last_undegraded":"2024-09-13T15:08:21.767476+0000","last_fullsized":"2024-09-13T15:08:21.767476+0000","mapping_epoch":395,"log_start":"403'5719","ondisk_log_start":"403'5719","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:20:15.405086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223215616,"num_objects":3406,"num_object_clones":0,"num_object_copies":10218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12164,"num_write_kb":498240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":542,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180192+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:08:10.180192+0000","last_peered":"2024-09-13T15:08:10.180192+0000","last_clean":"2024-09-13T15:08:10.180192+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:08:10.180192+0000","last_undegraded":"2024-09-13T15:08:10.180192+0000","last_fullsized":"2024-09-13T15:08:10.180192+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":718,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.243517+0000","last_change":"2024-09-13T15:08:09.187080+0000","last_active":"2024-09-13T15:08:20.243517+0000","last_peered":"2024-09-13T15:08:20.243517+0000","last_clean":"2024-09-13T15:08:20.243517+0000","last_became_active":"2024-09-13T15:08:09.186862+0000","last_became_peered":"2024-09-13T15:08:09.186862+0000","last_unstale":"2024-09-13T15:08:20.243517+0000","last_undegraded":"2024-09-13T15:08:20.243517+0000","last_fullsized":"2024-09-13T15:08:20.243517+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:59.089697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.236230+0000","last_change":"2024-09-13T15:07:52.946957+0000","last_active":"2024-09-13T15:08:20.236230+0000","last_peered":"2024-09-13T15:08:20.236230+0000","last_clean":"2024-09-13T15:08:20.236230+0000","last_became_active":"2024-09-13T15:07:52.946389+0000","last_became_peered":"2024-09-13T15:07:52.946389+0000","last_unstale":"2024-09-13T15:08:20.236230+0000","last_undegraded":"2024-09-13T15:08:20.236230+0000","last_fullsized":"2024-09-13T15:08:20.236230+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.963362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"433'12161","reported_seq":14286,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:22.613957+0000","last_change":"2024-09-13T15:07:52.974299+0000","last_active":"2024-09-13T15:08:22.613957+0000","last_peered":"2024-09-13T15:08:22.613957+0000","last_clean":"2024-09-13T15:08:22.613957+0000","last_became_active":"2024-09-13T15:07:52.956988+0000","last_became_peered":"2024-09-13T15:07:52.956988+0000","last_unstale":"2024-09-13T15:08:22.613957+0000","last_undegraded":"2024-09-13T15:08:22.613957+0000","last_fullsized":"2024-09-13T15:08:22.613957+0000","mapping_epoch":422,"log_start":"405'6706","ondisk_log_start":"405'6706","created":395,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:38.296775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12161,"num_write_kb":492672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1010,"num_bytes_recovered":66125824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":677,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179527+0000","last_change":"2024-09-13T15:08:09.189415+0000","last_active":"2024-09-13T15:08:10.179527+0000","last_peered":"2024-09-13T15:08:10.179527+0000","last_clean":"2024-09-13T15:08:10.179527+0000","last_became_active":"2024-09-13T15:08:09.188726+0000","last_became_peered":"2024-09-13T15:08:09.188726+0000","last_unstale":"2024-09-13T15:08:10.179527+0000","last_undegraded":"2024-09-13T15:08:10.179527+0000","last_fullsized":"2024-09-13T15:08:10.179527+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:00.397088+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":180,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180661+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:08:10.180661+0000","last_peered":"2024-09-13T15:08:10.180661+0000","last_clean":"2024-09-13T15:08:10.180661+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:08:10.180661+0000","last_undegraded":"2024-09-13T15:08:10.180661+0000","last_fullsized":"2024-09-13T15:08:10.180661+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":683,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.249326+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:08:20.249326+0000","last_peered":"2024-09-13T15:08:20.249326+0000","last_clean":"2024-09-13T15:08:20.249326+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:08:20.249326+0000","last_undegraded":"2024-09-13T15:08:20.249326+0000","last_fullsized":"2024-09-13T15:08:20.249326+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":419,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179923+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:08:10.179923+0000","last_peered":"2024-09-13T15:08:10.179923+0000","last_clean":"2024-09-13T15:08:10.179923+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:08:10.179923+0000","last_undegraded":"2024-09-13T15:08:10.179923+0000","last_fullsized":"2024-09-13T15:08:10.179923+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"433'12253","reported_seq":12301,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.725655+0000","last_change":"2024-09-13T15:05:11.104165+0000","last_active":"2024-09-13T15:08:21.725655+0000","last_peered":"2024-09-13T15:08:21.725655+0000","last_clean":"2024-09-13T15:08:21.725655+0000","last_became_active":"2024-09-13T15:05:11.103885+0000","last_became_peered":"2024-09-13T15:05:11.103885+0000","last_unstale":"2024-09-13T15:08:21.725655+0000","last_undegraded":"2024-09-13T15:08:21.725655+0000","last_fullsized":"2024-09-13T15:08:21.725655+0000","mapping_epoch":395,"log_start":"404'5806","ondisk_log_start":"404'5806","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6447,"log_dups_size":0,"ondisk_log_size":6447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:32.640246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214499328,"num_objects":3273,"num_object_clones":0,"num_object_copies":9819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3273,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12253,"num_write_kb":496832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":775,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.235753+0000","last_change":"2024-09-13T15:07:55.932598+0000","last_active":"2024-09-13T15:08:20.235753+0000","last_peered":"2024-09-13T15:08:20.235753+0000","last_clean":"2024-09-13T15:08:20.235753+0000","last_became_active":"2024-09-13T15:07:55.932277+0000","last_became_peered":"2024-09-13T15:07:55.932277+0000","last_unstale":"2024-09-13T15:08:20.235753+0000","last_undegraded":"2024-09-13T15:08:20.235753+0000","last_fullsized":"2024-09-13T15:08:20.235753+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:56.613460+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":635,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.239811+0000","last_change":"2024-09-13T15:06:40.607795+0000","last_active":"2024-09-13T15:08:20.239811+0000","last_peered":"2024-09-13T15:08:20.239811+0000","last_clean":"2024-09-13T15:08:20.239811+0000","last_became_active":"2024-09-13T15:06:40.607431+0000","last_became_peered":"2024-09-13T15:06:40.607431+0000","last_unstale":"2024-09-13T15:08:20.239811+0000","last_undegraded":"2024-09-13T15:08:20.239811+0000","last_fullsized":"2024-09-13T15:08:20.239811+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:47.696723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":226,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592241+0000","last_change":"2024-09-13T15:03:44.115491+0000","last_active":"2024-09-13T15:08:10.592241+0000","last_peered":"2024-09-13T15:08:10.592241+0000","last_clean":"2024-09-13T15:08:10.592241+0000","last_became_active":"2024-09-13T15:03:44.114978+0000","last_became_peered":"2024-09-13T15:03:44.114978+0000","last_unstale":"2024-09-13T15:08:10.592241+0000","last_undegraded":"2024-09-13T15:08:10.592241+0000","last_fullsized":"2024-09-13T15:08:10.592241+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:04.996891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.236518+0000","last_change":"2024-09-13T15:07:52.944954+0000","last_active":"2024-09-13T15:08:20.236518+0000","last_peered":"2024-09-13T15:08:20.236518+0000","last_clean":"2024-09-13T15:08:20.236518+0000","last_became_active":"2024-09-13T15:07:52.944654+0000","last_became_peered":"2024-09-13T15:07:52.944654+0000","last_unstale":"2024-09-13T15:08:20.236518+0000","last_undegraded":"2024-09-13T15:08:20.236518+0000","last_fullsized":"2024-09-13T15:08:20.236518+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:50.580294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"433'12180","reported_seq":12229,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.864024+0000","last_change":"2024-09-13T15:05:11.537213+0000","last_active":"2024-09-13T15:08:21.864024+0000","last_peered":"2024-09-13T15:08:21.864024+0000","last_clean":"2024-09-13T15:08:21.864024+0000","last_became_active":"2024-09-13T15:05:11.536909+0000","last_became_peered":"2024-09-13T15:05:11.536909+0000","last_unstale":"2024-09-13T15:08:21.864024+0000","last_undegraded":"2024-09-13T15:08:21.864024+0000","last_fullsized":"2024-09-13T15:08:21.864024+0000","mapping_epoch":395,"log_start":"403'5571","ondisk_log_start":"403'5571","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6609,"log_dups_size":0,"ondisk_log_size":6609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:13.844350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224198680,"num_objects":3422,"num_object_clones":0,"num_object_copies":10266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3422,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12180,"num_write_kb":499201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":682,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592008+0000","last_change":"2024-09-13T15:03:46.571609+0000","last_active":"2024-09-13T15:08:10.592008+0000","last_peered":"2024-09-13T15:08:10.592008+0000","last_clean":"2024-09-13T15:08:10.592008+0000","last_became_active":"2024-09-13T15:03:46.571111+0000","last_became_peered":"2024-09-13T15:03:46.571111+0000","last_unstale":"2024-09-13T15:08:10.592008+0000","last_undegraded":"2024-09-13T15:08:10.592008+0000","last_fullsized":"2024-09-13T15:08:10.592008+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:26.923160+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":765,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181356+0000","last_change":"2024-09-13T15:07:52.935042+0000","last_active":"2024-09-13T15:08:10.181356+0000","last_peered":"2024-09-13T15:08:10.181356+0000","last_clean":"2024-09-13T15:08:10.181356+0000","last_became_active":"2024-09-13T15:07:52.934697+0000","last_became_peered":"2024-09-13T15:07:52.934697+0000","last_unstale":"2024-09-13T15:08:10.181356+0000","last_undegraded":"2024-09-13T15:08:10.181356+0000","last_fullsized":"2024-09-13T15:08:10.181356+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:23.505991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.226959+0000","last_change":"2024-09-13T15:07:52.938622+0000","last_active":"2024-09-13T15:08:20.226959+0000","last_peered":"2024-09-13T15:08:20.226959+0000","last_clean":"2024-09-13T15:08:20.226959+0000","last_became_active":"2024-09-13T15:07:52.938323+0000","last_became_peered":"2024-09-13T15:07:52.938323+0000","last_unstale":"2024-09-13T15:08:20.226959+0000","last_undegraded":"2024-09-13T15:08:20.226959+0000","last_fullsized":"2024-09-13T15:08:20.226959+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:29.800644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":64,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.244910+0000","last_change":"2024-09-13T15:06:40.606903+0000","last_active":"2024-09-13T15:08:20.244910+0000","last_peered":"2024-09-13T15:08:20.244910+0000","last_clean":"2024-09-13T15:08:20.244910+0000","last_became_active":"2024-09-13T15:06:40.606596+0000","last_became_peered":"2024-09-13T15:06:40.606596+0000","last_unstale":"2024-09-13T15:08:20.244910+0000","last_undegraded":"2024-09-13T15:08:20.244910+0000","last_fullsized":"2024-09-13T15:08:20.244910+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:44.157857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.e","version":"433'11740","reported_seq":11787,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.792156+0000","last_change":"2024-09-13T15:05:11.104779+0000","last_active":"2024-09-13T15:08:21.792156+0000","last_peered":"2024-09-13T15:08:21.792156+0000","last_clean":"2024-09-13T15:08:21.792156+0000","last_became_active":"2024-09-13T15:05:11.104444+0000","last_became_peered":"2024-09-13T15:05:11.104444+0000","last_unstale":"2024-09-13T15:08:21.792156+0000","last_undegraded":"2024-09-13T15:08:21.792156+0000","last_fullsized":"2024-09-13T15:08:21.792156+0000","mapping_epoch":395,"log_start":"403'5289","ondisk_log_start":"403'5289","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:23.452855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215482368,"num_objects":3288,"num_object_clones":0,"num_object_copies":9864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11740,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":505,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181117+0000","last_change":"2024-09-13T15:03:47.227943+0000","last_active":"2024-09-13T15:08:10.181117+0000","last_peered":"2024-09-13T15:08:10.181117+0000","last_clean":"2024-09-13T15:08:10.181117+0000","last_became_active":"2024-09-13T15:03:46.585701+0000","last_became_peered":"2024-09-13T15:03:46.585701+0000","last_unstale":"2024-09-13T15:08:10.181117+0000","last_undegraded":"2024-09-13T15:08:10.181117+0000","last_fullsized":"2024-09-13T15:08:10.181117+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:03:47.226199+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:03:47.226199+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:12.460412+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":29,"reported_epoch":433,"state":"active+recovery_wait+degraded","last_fresh":"2024-09-13T15:08:20.231336+0000","last_change":"2024-09-13T15:07:52.940920+0000","last_active":"2024-09-13T15:08:20.231336+0000","last_peered":"2024-09-13T15:08:20.231336+0000","last_clean":"2024-09-13T14:19:32.222943+0000","last_became_active":"2024-09-13T15:07:52.939036+0000","last_became_peered":"2024-09-13T15:07:52.939036+0000","last_unstale":"2024-09-13T15:08:20.231336+0000","last_undegraded":"2024-09-13T15:07:52.921547+0000","last_fullsized":"2024-09-13T15:08:20.231336+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":169,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.593137+0000","last_change":"2024-09-13T15:08:09.186888+0000","last_active":"2024-09-13T15:08:10.593137+0000","last_peered":"2024-09-13T15:08:10.593137+0000","last_clean":"2024-09-13T15:08:10.593137+0000","last_became_active":"2024-09-13T15:08:09.186416+0000","last_became_peered":"2024-09-13T15:08:09.186416+0000","last_unstale":"2024-09-13T15:08:10.593137+0000","last_undegraded":"2024-09-13T15:08:10.593137+0000","last_fullsized":"2024-09-13T15:08:10.593137+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:56.667822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181399+0000","last_change":"2024-09-13T15:06:40.605296+0000","last_active":"2024-09-13T15:08:10.181399+0000","last_peered":"2024-09-13T15:08:10.181399+0000","last_clean":"2024-09-13T15:08:10.181399+0000","last_became_active":"2024-09-13T15:06:40.604969+0000","last_became_peered":"2024-09-13T15:06:40.604969+0000","last_unstale":"2024-09-13T15:08:10.181399+0000","last_undegraded":"2024-09-13T15:08:10.181399+0000","last_fullsized":"2024-09-13T15:08:10.181399+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:00.071208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.8","version":"432'11847","reported_seq":11987,"reported_epoch":432,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T15:08:18.273059+0000","last_change":"2024-09-13T15:07:03.386115+0000","last_active":"2024-09-13T15:08:18.273059+0000","last_peered":"2024-09-13T15:08:18.273059+0000","last_clean":"2024-09-13T15:06:40.084226+0000","last_became_active":"2024-09-13T15:06:41.880237+0000","last_became_peered":"2024-09-13T15:06:41.880237+0000","last_unstale":"2024-09-13T15:08:18.273059+0000","last_undegraded":"2024-09-13T15:08:18.273059+0000","last_fullsized":"2024-09-13T15:08:18.273059+0000","mapping_epoch":407,"log_start":"402'4700","ondisk_log_start":"402'4700","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7147,"log_dups_size":0,"ondisk_log_size":7147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3553,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11847,"num_write_kb":493120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1245184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":3563}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":686,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180060+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:08:10.180060+0000","last_peered":"2024-09-13T15:08:10.180060+0000","last_clean":"2024-09-13T15:08:10.180060+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:08:10.180060+0000","last_undegraded":"2024-09-13T15:08:10.180060+0000","last_fullsized":"2024-09-13T15:08:10.180060+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.253033+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:08:20.253033+0000","last_peered":"2024-09-13T15:08:20.253033+0000","last_clean":"2024-09-13T15:08:20.253033+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:08:20.253033+0000","last_undegraded":"2024-09-13T15:08:20.253033+0000","last_fullsized":"2024-09-13T15:08:20.253033+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.236031+0000","last_change":"2024-09-13T15:07:52.933730+0000","last_active":"2024-09-13T15:08:20.236031+0000","last_peered":"2024-09-13T15:08:20.236031+0000","last_clean":"2024-09-13T15:08:20.236031+0000","last_became_active":"2024-09-13T15:07:52.931809+0000","last_became_peered":"2024-09-13T15:07:52.931809+0000","last_unstale":"2024-09-13T15:08:20.236031+0000","last_undegraded":"2024-09-13T15:08:20.236031+0000","last_fullsized":"2024-09-13T15:08:20.236031+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:23.226452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":444,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591747+0000","last_change":"2024-09-13T15:06:40.601744+0000","last_active":"2024-09-13T15:08:10.591747+0000","last_peered":"2024-09-13T15:08:10.591747+0000","last_clean":"2024-09-13T15:08:10.591747+0000","last_became_active":"2024-09-13T15:06:40.601439+0000","last_became_peered":"2024-09-13T15:06:40.601439+0000","last_unstale":"2024-09-13T15:08:10.591747+0000","last_undegraded":"2024-09-13T15:08:10.591747+0000","last_fullsized":"2024-09-13T15:08:10.591747+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:47.746856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.9","version":"433'12161","reported_seq":12248,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:22.614549+0000","last_change":"2024-09-13T15:05:11.537327+0000","last_active":"2024-09-13T15:08:22.614549+0000","last_peered":"2024-09-13T15:08:22.614549+0000","last_clean":"2024-09-13T15:08:22.614549+0000","last_became_active":"2024-09-13T15:05:11.536882+0000","last_became_peered":"2024-09-13T15:05:11.536882+0000","last_unstale":"2024-09-13T15:08:22.614549+0000","last_undegraded":"2024-09-13T15:08:22.614549+0000","last_fullsized":"2024-09-13T15:08:22.614549+0000","mapping_epoch":395,"log_start":"405'6698","ondisk_log_start":"405'6698","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:53:40.934767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212533248,"num_objects":3243,"num_object_clones":0,"num_object_copies":9729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12161,"num_write_kb":492928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":752,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179928+0000","last_change":"2024-09-13T15:03:46.574886+0000","last_active":"2024-09-13T15:08:10.179928+0000","last_peered":"2024-09-13T15:08:10.179928+0000","last_clean":"2024-09-13T15:08:10.179928+0000","last_became_active":"2024-09-13T15:03:46.574563+0000","last_became_peered":"2024-09-13T15:03:46.574563+0000","last_unstale":"2024-09-13T15:08:10.179928+0000","last_undegraded":"2024-09-13T15:08:10.179928+0000","last_fullsized":"2024-09-13T15:08:10.179928+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:29.856598+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":105,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181299+0000","last_change":"2024-09-13T15:08:09.186120+0000","last_active":"2024-09-13T15:08:10.181299+0000","last_peered":"2024-09-13T15:08:10.181299+0000","last_clean":"2024-09-13T15:08:10.181299+0000","last_became_active":"2024-09-13T15:08:09.185851+0000","last_became_peered":"2024-09-13T15:08:09.185851+0000","last_unstale":"2024-09-13T15:08:10.181299+0000","last_undegraded":"2024-09-13T15:08:10.181299+0000","last_fullsized":"2024-09-13T15:08:10.181299+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:22.561596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":253,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.595438+0000","last_change":"2024-09-13T15:08:09.187714+0000","last_active":"2024-09-13T15:08:10.595438+0000","last_peered":"2024-09-13T15:08:10.595438+0000","last_clean":"2024-09-13T15:08:10.595438+0000","last_became_active":"2024-09-13T15:08:09.187404+0000","last_became_peered":"2024-09-13T15:08:09.187404+0000","last_unstale":"2024-09-13T15:08:10.595438+0000","last_undegraded":"2024-09-13T15:08:10.595438+0000","last_fullsized":"2024-09-13T15:08:10.595438+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:04.738653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.235851+0000","last_change":"2024-09-13T15:07:52.933604+0000","last_active":"2024-09-13T15:08:20.235851+0000","last_peered":"2024-09-13T15:08:20.235851+0000","last_clean":"2024-09-13T15:08:20.235851+0000","last_became_active":"2024-09-13T15:07:52.933192+0000","last_became_peered":"2024-09-13T15:07:52.933192+0000","last_unstale":"2024-09-13T15:08:20.235851+0000","last_undegraded":"2024-09-13T15:08:20.235851+0000","last_fullsized":"2024-09-13T15:08:20.235851+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:05.303014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.a","version":"433'12373","reported_seq":12466,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:21.969377+0000","last_change":"2024-09-13T15:07:53.951600+0000","last_active":"2024-09-13T15:08:21.969377+0000","last_peered":"2024-09-13T15:08:21.969377+0000","last_clean":"2024-09-13T15:08:21.969377+0000","last_became_active":"2024-09-13T15:07:53.925119+0000","last_became_peered":"2024-09-13T15:07:53.925119+0000","last_unstale":"2024-09-13T15:08:21.969377+0000","last_undegraded":"2024-09-13T15:08:21.969377+0000","last_fullsized":"2024-09-13T15:08:21.969377+0000","mapping_epoch":423,"log_start":"403'5621","ondisk_log_start":"403'5621","created":395,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6752,"log_dups_size":0,"ondisk_log_size":6752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:39:08.129515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222494720,"num_objects":3395,"num_object_clones":0,"num_object_copies":10185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12373,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":615,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.595362+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:08:10.595362+0000","last_peered":"2024-09-13T15:08:10.595362+0000","last_clean":"2024-09-13T15:08:10.595362+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:08:10.595362+0000","last_undegraded":"2024-09-13T15:08:10.595362+0000","last_fullsized":"2024-09-13T15:08:10.595362+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":727,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.245450+0000","last_change":"2024-09-13T15:03:44.131836+0000","last_active":"2024-09-13T15:08:20.245450+0000","last_peered":"2024-09-13T15:08:20.245450+0000","last_clean":"2024-09-13T15:08:20.245450+0000","last_became_active":"2024-09-13T15:03:44.120633+0000","last_became_peered":"2024-09-13T15:03:44.120633+0000","last_unstale":"2024-09-13T15:08:20.245450+0000","last_undegraded":"2024-09-13T15:08:20.245450+0000","last_fullsized":"2024-09-13T15:08:20.245450+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:52.065604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":519,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180146+0000","last_change":"2024-09-13T15:06:40.605165+0000","last_active":"2024-09-13T15:08:10.180146+0000","last_peered":"2024-09-13T15:08:10.180146+0000","last_clean":"2024-09-13T15:08:10.180146+0000","last_became_active":"2024-09-13T15:06:40.604503+0000","last_became_peered":"2024-09-13T15:06:40.604503+0000","last_unstale":"2024-09-13T15:08:10.180146+0000","last_undegraded":"2024-09-13T15:08:10.180146+0000","last_fullsized":"2024-09-13T15:08:10.180146+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:57.829820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":741,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.243355+0000","last_change":"2024-09-13T15:07:52.933408+0000","last_active":"2024-09-13T15:08:20.243355+0000","last_peered":"2024-09-13T15:08:20.243355+0000","last_clean":"2024-09-13T15:08:20.243355+0000","last_became_active":"2024-09-13T15:07:52.933015+0000","last_became_peered":"2024-09-13T15:07:52.933015+0000","last_unstale":"2024-09-13T15:08:20.243355+0000","last_undegraded":"2024-09-13T15:08:20.243355+0000","last_fullsized":"2024-09-13T15:08:20.243355+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:51.373304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"432'11820","reported_seq":11925,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:18.278183+0000","last_change":"2024-09-13T15:07:53.953162+0000","last_active":"2024-09-13T15:08:18.278183+0000","last_peered":"2024-09-13T15:08:18.278183+0000","last_clean":"2024-09-13T15:08:18.278183+0000","last_became_active":"2024-09-13T15:07:53.926761+0000","last_became_peered":"2024-09-13T15:07:53.926761+0000","last_unstale":"2024-09-13T15:08:18.278183+0000","last_undegraded":"2024-09-13T15:08:18.278183+0000","last_fullsized":"2024-09-13T15:08:18.278183+0000","mapping_epoch":423,"log_start":"402'4645","ondisk_log_start":"402'4645","created":395,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7175,"log_dups_size":0,"ondisk_log_size":7175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:27.715834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237764608,"num_objects":3628,"num_object_clones":0,"num_object_copies":10884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11820,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":957,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.243978+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:08:20.243978+0000","last_peered":"2024-09-13T15:08:20.243978+0000","last_clean":"2024-09-13T15:08:20.243978+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:08:20.243978+0000","last_undegraded":"2024-09-13T15:08:20.243978+0000","last_fullsized":"2024-09-13T15:08:20.243978+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":337,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.593135+0000","last_change":"2024-09-13T15:08:09.190731+0000","last_active":"2024-09-13T15:08:10.593135+0000","last_peered":"2024-09-13T15:08:10.593135+0000","last_clean":"2024-09-13T15:08:10.593135+0000","last_became_active":"2024-09-13T15:08:09.187657+0000","last_became_peered":"2024-09-13T15:08:09.187657+0000","last_unstale":"2024-09-13T15:08:10.593135+0000","last_undegraded":"2024-09-13T15:08:10.593135+0000","last_fullsized":"2024-09-13T15:08:10.593135+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:44.137504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":423,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180148+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:08:10.180148+0000","last_peered":"2024-09-13T15:08:10.180148+0000","last_clean":"2024-09-13T15:08:10.180148+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:08:10.180148+0000","last_undegraded":"2024-09-13T15:08:10.180148+0000","last_fullsized":"2024-09-13T15:08:10.180148+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":122,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592310+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:08:10.592310+0000","last_peered":"2024-09-13T15:08:10.592310+0000","last_clean":"2024-09-13T15:08:10.592310+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:08:10.592310+0000","last_undegraded":"2024-09-13T15:08:10.592310+0000","last_fullsized":"2024-09-13T15:08:10.592310+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":148,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.593207+0000","last_change":"2024-09-13T15:08:09.188203+0000","last_active":"2024-09-13T15:08:10.593207+0000","last_peered":"2024-09-13T15:08:10.593207+0000","last_clean":"2024-09-13T15:08:10.593207+0000","last_became_active":"2024-09-13T15:08:09.187271+0000","last_became_peered":"2024-09-13T15:08:09.187271+0000","last_unstale":"2024-09-13T15:08:10.593207+0000","last_undegraded":"2024-09-13T15:08:10.593207+0000","last_fullsized":"2024-09-13T15:08:10.593207+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:31.670696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":131,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591711+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:08:10.591711+0000","last_peered":"2024-09-13T15:08:10.591711+0000","last_clean":"2024-09-13T15:08:10.591711+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:08:10.591711+0000","last_undegraded":"2024-09-13T15:08:10.591711+0000","last_fullsized":"2024-09-13T15:08:10.591711+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":115,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179628+0000","last_change":"2024-09-13T15:06:40.601737+0000","last_active":"2024-09-13T15:08:10.179628+0000","last_peered":"2024-09-13T15:08:10.179628+0000","last_clean":"2024-09-13T15:08:10.179628+0000","last_became_active":"2024-09-13T15:06:40.601141+0000","last_became_peered":"2024-09-13T15:06:40.601141+0000","last_unstale":"2024-09-13T15:08:10.179628+0000","last_undegraded":"2024-09-13T15:08:10.179628+0000","last_fullsized":"2024-09-13T15:08:10.179628+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:56.285800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":668,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181050+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:08:10.181050+0000","last_peered":"2024-09-13T15:08:10.181050+0000","last_clean":"2024-09-13T15:08:10.181050+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:08:10.181050+0000","last_undegraded":"2024-09-13T15:08:10.181050+0000","last_fullsized":"2024-09-13T15:08:10.181050+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":64,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.241464+0000","last_change":"2024-09-13T15:06:40.606724+0000","last_active":"2024-09-13T15:08:20.241464+0000","last_peered":"2024-09-13T15:08:20.241464+0000","last_clean":"2024-09-13T15:08:20.241464+0000","last_became_active":"2024-09-13T15:06:40.606228+0000","last_became_peered":"2024-09-13T15:06:40.606228+0000","last_unstale":"2024-09-13T15:08:20.241464+0000","last_undegraded":"2024-09-13T15:08:20.241464+0000","last_fullsized":"2024-09-13T15:08:20.241464+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:21.085510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.179363+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:08:10.179363+0000","last_peered":"2024-09-13T15:08:10.179363+0000","last_clean":"2024-09-13T15:08:10.179363+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:08:10.179363+0000","last_undegraded":"2024-09-13T15:08:10.179363+0000","last_fullsized":"2024-09-13T15:08:10.179363+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":322,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.245287+0000","last_change":"2024-09-13T15:03:44.113646+0000","last_active":"2024-09-13T15:08:20.245287+0000","last_peered":"2024-09-13T15:08:20.245287+0000","last_clean":"2024-09-13T15:08:20.245287+0000","last_became_active":"2024-09-13T15:03:44.113342+0000","last_became_peered":"2024-09-13T15:03:44.113342+0000","last_unstale":"2024-09-13T15:08:20.245287+0000","last_undegraded":"2024-09-13T15:08:20.245287+0000","last_fullsized":"2024-09-13T15:08:20.245287+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:29.444600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":715,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.243270+0000","last_change":"2024-09-13T15:08:09.184381+0000","last_active":"2024-09-13T15:08:20.243270+0000","last_peered":"2024-09-13T15:08:20.243270+0000","last_clean":"2024-09-13T15:08:20.243270+0000","last_became_active":"2024-09-13T15:08:09.184077+0000","last_became_peered":"2024-09-13T15:08:09.184077+0000","last_unstale":"2024-09-13T15:08:20.243270+0000","last_undegraded":"2024-09-13T15:08:20.243270+0000","last_fullsized":"2024-09-13T15:08:20.243270+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:26.703635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":428,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180137+0000","last_change":"2024-09-13T15:07:52.936799+0000","last_active":"2024-09-13T15:08:10.180137+0000","last_peered":"2024-09-13T15:08:10.180137+0000","last_clean":"2024-09-13T15:08:10.180137+0000","last_became_active":"2024-09-13T15:07:52.936436+0000","last_became_peered":"2024-09-13T15:07:52.936436+0000","last_unstale":"2024-09-13T15:08:10.180137+0000","last_undegraded":"2024-09-13T15:08:10.180137+0000","last_fullsized":"2024-09-13T15:08:10.180137+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:52:09.480999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":763,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.243430+0000","last_change":"2024-09-13T15:06:40.608166+0000","last_active":"2024-09-13T15:08:20.243430+0000","last_peered":"2024-09-13T15:08:20.243430+0000","last_clean":"2024-09-13T15:08:20.243430+0000","last_became_active":"2024-09-13T15:06:40.607830+0000","last_became_peered":"2024-09-13T15:06:40.607830+0000","last_unstale":"2024-09-13T15:08:20.243430+0000","last_undegraded":"2024-09-13T15:08:20.243430+0000","last_fullsized":"2024-09-13T15:08:20.243430+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:48.177335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":479,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.248667+0000","last_change":"2024-09-13T15:07:52.941232+0000","last_active":"2024-09-13T15:08:20.248667+0000","last_peered":"2024-09-13T15:08:20.248667+0000","last_clean":"2024-09-13T15:08:20.248667+0000","last_became_active":"2024-09-13T15:07:52.939661+0000","last_became_peered":"2024-09-13T15:07:52.939661+0000","last_unstale":"2024-09-13T15:08:20.248667+0000","last_undegraded":"2024-09-13T15:08:20.248667+0000","last_fullsized":"2024-09-13T15:08:20.248667+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:42.857066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":32,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.230527+0000","last_change":"2024-09-13T15:07:52.936887+0000","last_active":"2024-09-13T15:08:20.230527+0000","last_peered":"2024-09-13T15:08:20.230527+0000","last_clean":"2024-09-13T15:08:20.230527+0000","last_became_active":"2024-09-13T15:07:52.936398+0000","last_became_peered":"2024-09-13T15:07:52.936398+0000","last_unstale":"2024-09-13T15:08:20.230527+0000","last_undegraded":"2024-09-13T15:08:20.230527+0000","last_fullsized":"2024-09-13T15:08:20.230527+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:05.892375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":722,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.252695+0000","last_change":"2024-09-13T15:07:52.935989+0000","last_active":"2024-09-13T15:08:20.252695+0000","last_peered":"2024-09-13T15:08:20.252695+0000","last_clean":"2024-09-13T15:08:20.252695+0000","last_became_active":"2024-09-13T15:07:52.935661+0000","last_became_peered":"2024-09-13T15:07:52.935661+0000","last_unstale":"2024-09-13T15:08:20.252695+0000","last_undegraded":"2024-09-13T15:08:20.252695+0000","last_fullsized":"2024-09-13T15:08:20.252695+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:49.027141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":139,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591908+0000","last_change":"2024-09-13T15:07:52.935604+0000","last_active":"2024-09-13T15:08:10.591908+0000","last_peered":"2024-09-13T15:08:10.591908+0000","last_clean":"2024-09-13T15:08:10.591908+0000","last_became_active":"2024-09-13T15:07:52.932711+0000","last_became_peered":"2024-09-13T15:07:52.932711+0000","last_unstale":"2024-09-13T15:08:10.591908+0000","last_undegraded":"2024-09-13T15:08:10.591908+0000","last_fullsized":"2024-09-13T15:08:10.591908+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:06.384328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":732,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.181341+0000","last_change":"2024-09-13T15:07:52.931529+0000","last_active":"2024-09-13T15:08:10.181341+0000","last_peered":"2024-09-13T15:08:10.181341+0000","last_clean":"2024-09-13T15:08:10.181341+0000","last_became_active":"2024-09-13T15:07:52.931055+0000","last_became_peered":"2024-09-13T15:07:52.931055+0000","last_unstale":"2024-09-13T15:08:10.181341+0000","last_undegraded":"2024-09-13T15:08:10.181341+0000","last_fullsized":"2024-09-13T15:08:10.181341+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:02.788954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":116,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.593001+0000","last_change":"2024-09-13T15:06:40.605841+0000","last_active":"2024-09-13T15:08:10.593001+0000","last_peered":"2024-09-13T15:08:10.593001+0000","last_clean":"2024-09-13T15:08:10.593001+0000","last_became_active":"2024-09-13T15:06:40.604704+0000","last_became_peered":"2024-09-13T15:06:40.604704+0000","last_unstale":"2024-09-13T15:08:10.593001+0000","last_undegraded":"2024-09-13T15:08:10.593001+0000","last_fullsized":"2024-09-13T15:08:10.593001+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:58.305980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":139,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.592913+0000","last_change":"2024-09-13T15:06:40.603223+0000","last_active":"2024-09-13T15:08:10.592913+0000","last_peered":"2024-09-13T15:08:10.592913+0000","last_clean":"2024-09-13T15:08:10.592913+0000","last_became_active":"2024-09-13T15:06:40.602930+0000","last_became_peered":"2024-09-13T15:06:40.602930+0000","last_unstale":"2024-09-13T15:08:10.592913+0000","last_undegraded":"2024-09-13T15:08:10.592913+0000","last_fullsized":"2024-09-13T15:08:10.592913+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:16.757274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":429,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591634+0000","last_change":"2024-09-13T15:07:52.943245+0000","last_active":"2024-09-13T15:08:10.591634+0000","last_peered":"2024-09-13T15:08:10.591634+0000","last_clean":"2024-09-13T15:08:10.591634+0000","last_became_active":"2024-09-13T15:07:52.932624+0000","last_became_peered":"2024-09-13T15:07:52.932624+0000","last_unstale":"2024-09-13T15:08:10.591634+0000","last_undegraded":"2024-09-13T15:08:10.591634+0000","last_fullsized":"2024-09-13T15:08:10.591634+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:50.061001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":141,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591818+0000","last_change":"2024-09-13T15:03:44.116627+0000","last_active":"2024-09-13T15:08:10.591818+0000","last_peered":"2024-09-13T15:08:10.591818+0000","last_clean":"2024-09-13T15:08:10.591818+0000","last_became_active":"2024-09-13T15:03:44.116197+0000","last_became_peered":"2024-09-13T15:03:44.116197+0000","last_unstale":"2024-09-13T15:08:10.591818+0000","last_undegraded":"2024-09-13T15:08:10.591818+0000","last_fullsized":"2024-09-13T15:08:10.591818+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:30.519779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":388,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.595277+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:08:10.595277+0000","last_peered":"2024-09-13T15:08:10.595277+0000","last_clean":"2024-09-13T15:08:10.595277+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:08:10.595277+0000","last_undegraded":"2024-09-13T15:08:10.595277+0000","last_fullsized":"2024-09-13T15:08:10.595277+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":758,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.253141+0000","last_change":"2024-09-13T15:03:44.120794+0000","last_active":"2024-09-13T15:08:20.253141+0000","last_peered":"2024-09-13T15:08:20.253141+0000","last_clean":"2024-09-13T15:08:20.253141+0000","last_became_active":"2024-09-13T15:03:44.120431+0000","last_became_peered":"2024-09-13T15:03:44.120431+0000","last_unstale":"2024-09-13T15:08:20.253141+0000","last_undegraded":"2024-09-13T15:08:20.253141+0000","last_fullsized":"2024-09-13T15:08:20.253141+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:27.396753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":717,"reported_epoch":433,"state":"active+clean","last_fresh":"2024-09-13T15:08:20.248827+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:08:20.248827+0000","last_peered":"2024-09-13T15:08:20.248827+0000","last_clean":"2024-09-13T15:08:20.248827+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:08:20.248827+0000","last_undegraded":"2024-09-13T15:08:20.248827+0000","last_fullsized":"2024-09-13T15:08:20.248827+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":139,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.591661+0000","last_change":"2024-09-13T15:07:52.933733+0000","last_active":"2024-09-13T15:08:10.591661+0000","last_peered":"2024-09-13T15:08:10.591661+0000","last_clean":"2024-09-13T15:08:10.591661+0000","last_became_active":"2024-09-13T15:07:52.933425+0000","last_became_peered":"2024-09-13T15:07:52.933425+0000","last_unstale":"2024-09-13T15:08:10.591661+0000","last_undegraded":"2024-09-13T15:08:10.591661+0000","last_fullsized":"2024-09-13T15:08:10.591661+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:59.564486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":151,"reported_epoch":432,"state":"active+clean","last_fresh":"2024-09-13T15:08:10.180679+0000","last_change":"2024-09-13T15:03:44.120052+0000","last_active":"2024-09-13T15:08:10.180679+0000","last_peered":"2024-09-13T15:08:10.180679+0000","last_clean":"2024-09-13T15:08:10.180679+0000","last_became_active":"2024-09-13T15:03:44.109611+0000","last_became_peered":"2024-09-13T15:03:44.109611+0000","last_unstale":"2024-09-13T15:08:10.180679+0000","last_undegraded":"2024-09-13T15:08:10.180679+0000","last_fullsized":"2024-09-13T15:08:10.180679+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:04:55.196794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":5808,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":16,"stat_sum":{"num_bytes":3561160728,"num_objects":54340,"num_object_clones":0,"num_object_copies":163020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2553,"num_objects_misplaced":9805,"num_objects_unfound":0,"num_objects_dirty":54340,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193570,"num_write_kb":7933057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16330,"num_bytes_recovered":1069940736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11254509568,"data_stored":11254497352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102861,"ondisk_log_size":102861,"up":48,"acting":47,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234579,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2282668,"kb_used_data":885796,"kb_used_omap":1,"kb_used_meta":1396862,"kb_avail":91462484,"statfs":{"total":95995035648,"available":93657583616,"internally_reserved":0,"allocated":907055104,"data_stored":904726754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1430387147},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496217,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968372,"kb_used_data":1573224,"kb_used_omap":1,"kb_used_meta":1395134,"kb_avail":90776780,"statfs":{"total":95995035648,"available":92955422720,"internally_reserved":0,"allocated":1610981376,"data_stored":1608649499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1428617671},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757855,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2958444,"kb_used_data":1338336,"kb_used_omap":1,"kb_used_meta":1620030,"kb_avail":90786708,"statfs":{"total":95995035648,"available":92965588992,"internally_reserved":0,"allocated":1370456064,"data_stored":1368127858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1658911174},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019491,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2832048,"kb_used_data":1527016,"kb_used_omap":1,"kb_used_meta":1305022,"kb_avail":90913104,"statfs":{"total":95995035648,"available":93095018496,"internally_reserved":0,"allocated":1563664384,"data_stored":1561331982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1336342982},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650703,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3240160,"kb_used_data":2143832,"kb_used_omap":4,"kb_used_meta":1096251,"kb_avail":90504992,"statfs":{"total":95995035648,"available":92677111808,"internally_reserved":0,"allocated":2195283968,"data_stored":2192963689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":1122561583},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879637,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678232,"kb_used_data":1606032,"kb_used_omap":6,"kb_used_meta":1072057,"kb_avail":91066920,"statfs":{"total":95995035648,"available":93252526080,"internally_reserved":0,"allocated":1644576768,"data_stored":1642256952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":1097786937},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435655,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3361680,"kb_used_data":2082116,"kb_used_omap":5,"kb_used_meta":1279546,"kb_avail":90383472,"statfs":{"total":95995035648,"available":92552675328,"internally_reserved":0,"allocated":2132086784,"data_stored":2129785786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1310255457},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":1494648619098,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2106916864,"data_stored":2106916864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1621622784,"data_stored":1621622784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2170880000,"data_stored":2170880000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1541672960,"data_stored":1541668888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1341263872,"data_stored":1341259800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1579094016,"data_stored":1579089944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":893059072,"data_stored":893059072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:08:26.127 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T15:08:26.128 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_9 pg_num 26 2024-09-13T15:08:26.132 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:26.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:26.235 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.301 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:26.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:26.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[113926]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[113926]: pgmap v1553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2730/159615 objects degraded (1.710%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[108570]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[108570]: pgmap v1553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:26 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2730/159615 objects degraded (1.710%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:26.402 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:26.427 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:26.489 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:26 smithi159 ceph-mon[102519]: from='client.46274 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:26.489 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:26 smithi159 ceph-mon[102519]: pgmap v1553: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:26.489 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.489 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:26.489 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:26 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2730/159615 objects degraded (1.710%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:26.529 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:26.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:26.670 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:26.710 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:26.812 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:26.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:26.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:26.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:26.920 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:08:26.920 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221521 12.0 12:056d5c1b:::benchmark_data_smithi042_260592_object104630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.046534+0000", 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.86799739000003, 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99136147299999999, 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.921 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221521 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046534+0000", 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046534+0000", 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046532+0000", 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046540+0000", 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2320000000000001e-06 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046563+0000", 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2138e-05 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046572+0000", 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2380000000000003e-06 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046640+0000", 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8200000000000004e-05 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.046651+0000", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0851e-05 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.028908+0000", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98225693999999997 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.029000+0000", 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.2590999999999995e-05 2024-09-13T15:08:26.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.029415+0000", 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00041501199999999998 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037665+0000", 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0082499570000000005 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037676+0000", 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0370999999999999e-05 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037805+0000", 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000129314 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037855+0000", 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9889999999999998e-05 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037896+0000", 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0737e-05 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221525 12.0 12:0693679d:::benchmark_data_smithi042_260592_object104634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.048236+0000", 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.86629547000001, 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99520446600000001, 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.928 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221525 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048236+0000", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048236+0000", 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048234+0000", 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048240+0000", 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9989999999999997e-06 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048252+0000", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1474000000000001e-05 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048259+0000", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7000000000000002e-06 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048298+0000", 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8930999999999997e-05 2024-09-13T15:08:26.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.048304+0000", 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1460000000000001e-06 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.032520+0000", 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98421641800000004 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.932 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.032655+0000", 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000134864 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.033142+0000", 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00048666399999999999 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037937+0000", 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047948660000000001 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037942+0000", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9609999999999996e-06 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043348+0000", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0054069369999999997 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043400+0000", 2024-09-13T15:08:26.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1326999999999998e-05 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043441+0000", 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0837000000000002e-05 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221580 12.0 12:016970c9:::benchmark_data_smithi042_260592_object104689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.098028+0000", 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.81650402299999, 2024-09-13T15:08:26.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.94560057500000005, 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221580 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098028+0000", 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098028+0000", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098024+0000", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999962 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098032+0000", 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.4840000000000004e-06 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098044+0000", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2169e-05 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098050+0000", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0399999999999998e-06 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098104+0000", 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4036999999999998e-05 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.098116+0000", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1797000000000001e-05 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.035080+0000", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.93696370900000003 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.035200+0000", 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000119726 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.035537+0000", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000337506 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037967+0000", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024299289999999999 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037974+0000", 2024-09-13T15:08:26.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.922e-06 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043552+0000", 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0055777989999999996 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043595+0000", 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3405000000000002e-05 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043628+0000", 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2913000000000001e-05 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221604 12.0 12:055652d4:::benchmark_data_smithi042_260592_object104713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.118680+0000", 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.79585126400002, 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.93203303699999995, 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.942 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221604 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118680+0000", 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118680+0000", 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118678+0000", 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118685+0000", 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6950000000000003e-06 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118696+0000", 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0957e-05 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118703+0000", 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8539999999999999e-06 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118763+0000", 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0368000000000002e-05 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.118773+0000", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7370000000000001e-06 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.037992+0000", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91921897600000002 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.038101+0000", 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109279 2024-09-13T15:08:26.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.038547+0000", 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00044581299999999998 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043495+0000", 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0049476859999999998 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043504+0000", 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0259999999999996e-06 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.050627+0000", 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071234810000000001 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.050674+0000", 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6965999999999997e-05 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.050713+0000", 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9110999999999999e-05 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.949 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221622 12.0 12:00b1b256:::benchmark_data_smithi042_260592_object104731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.134088+0000", 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.78044351099999, 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.92504482499999996, 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221622 2024-09-13T15:08:26.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134088+0000", 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134088+0000", 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134086+0000", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134093+0000", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0099999999999998e-06 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134107+0000", 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4153e-05 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134116+0000", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5089999999999993e-06 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134177+0000", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1278999999999994e-05 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.134197+0000", 2024-09-13T15:08:26.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9803999999999999e-05 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.040584+0000", 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90638670700000001 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.040704+0000", 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000120208 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.041055+0000", 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00035162399999999998 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043679+0000", 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0026237209999999999 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043686+0000", 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8770000000000004e-06 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059054+0000", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.015368389 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059098+0000", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3752999999999999e-05 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059133+0000", 2024-09-13T15:08:26.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.4790999999999999e-05 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45599.0:221631 12.0 12:07c10ad1:::benchmark_data_smithi042_260592_object104740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e336)", 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:00:07.143608+0000", 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 499.770923697, 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91565898700000004, 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.957 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45599", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/2985706518", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 221631 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143608+0000", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143608+0000", 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.958 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143606+0000", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143613+0000", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1629999999999999e-06 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143632+0000", 2024-09-13T15:08:26.959 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8547e-05 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143642+0000", 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0349999999999999e-05 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143715+0000", 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3052999999999993e-05 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:07.143724+0000", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0000000000000002e-06 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043717+0000", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89999272600000002 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.043834+0000", 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117133 2024-09-13T15:08:26.961 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 2,6", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.044259+0000", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00042466699999999999 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059009+0000", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.014749831999999999 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059016+0000", 2024-09-13T15:08:26.962 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9410000000000001e-06 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059197+0000", 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018175400000000001 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059236+0000", 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8868000000000001e-05 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.963 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:00:08.059267+0000", 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0660000000000001e-05 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60169 13.e 13:71178aff:::benchmark_data_smithi042_286876_object60168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 414.056246735, 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0134129039999999, 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.964 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60169 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.965 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858283+0000", 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858317+0000", 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3337999999999998e-05 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858325+0000", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5189999999999992e-06 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858331+0000", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7440000000000004e-06 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858652+0000", 2024-09-13T15:08:26.967 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000321031 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858664+0000", 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1865000000000001e-05 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.859035+0000", 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.000370776 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.968 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.860409+0000", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013744390000000001 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.860580+0000", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017043399999999999 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867875+0000", 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0072958679999999996 2024-09-13T15:08:26.969 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867909+0000", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3773999999999999e-05 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871610+0000", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037006669999999999 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871656+0000", 2024-09-13T15:08:26.970 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6369000000000003e-05 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871698+0000", 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1471999999999999e-05 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60199 13.e 13:7465222c:::benchmark_data_smithi042_286876_object60198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:08:26.971 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 414.02973396099998, 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98705518800000003, 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60199 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.972 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884797+0000", 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884856+0000", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9181999999999997e-05 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884867+0000", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0471e-05 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884873+0000", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7689999999999999e-06 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.974 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884932+0000", 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8471000000000003e-05 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884944+0000", 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2184999999999999e-05 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.862746+0000", 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97780187600000001 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.862928+0000", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000182528 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.863148+0000", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000219435 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867943+0000", 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047948289999999996 2024-09-13T15:08:26.976 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871732+0000", 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037891190000000001 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.977 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.977 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:26.978 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.978 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871779+0000", 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7583000000000002e-05 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871821+0000", 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1870999999999999e-05 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871853+0000", 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1609000000000002e-05 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.979 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60203 13.e 13:7b415ff7:::benchmark_data_smithi042_286876_object60202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 414.02744491599998, 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98575690599999999, 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:26.980 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60203 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887084+0000", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999971 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887151+0000", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7595000000000006e-05 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887187+0000", 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5825999999999999e-05 2024-09-13T15:08:26.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887195+0000", 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4139999999999997e-06 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887225+0000", 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0437e-05 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.983 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887232+0000", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.815e-06 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865319+0000", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97808694299999999 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865466+0000", 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014752600000000001 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.984 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865621+0000", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015456000000000001 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871737+0000", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061158360000000004 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872278+0000", 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000541349 2024-09-13T15:08:26.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872756+0000", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00047819699999999998 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872806+0000", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9539999999999997e-05 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872844+0000", 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7774000000000001e-05 2024-09-13T15:08:26.986 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213354 13.5 13:a90db7da:::benchmark_data_smithi042_286876_object89853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.90189699699999, 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.057659903, 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213354 2024-09-13T15:08:26.987 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012634+0000", 2024-09-13T15:08:26.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012636+0000", 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2019999999999998e-06 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012642+0000", 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2340000000000002e-06 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012646+0000", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1659999999999996e-06 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024265+0000", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.011619378 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024273+0000", 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9999999999999996e-06 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.990 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059337+0000", 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.035064153 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059442+0000", 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000104627 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:26.991 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059586+0000", 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014403699999999999 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063004+0000", 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003418291 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066853+0000", 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003848803 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070209+0000", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003355575 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070256+0000", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7079000000000001e-05 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070295+0000", 2024-09-13T15:08:26.993 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8770000000000003e-05 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213357 13.5 13:a5fa48f4:::benchmark_data_smithi042_286876_object89856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.89879760100001, 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.06210454, 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:26.994 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213357 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.995 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015733+0000", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015741+0000", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5000000000000002e-06 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015749+0000", 2024-09-13T15:08:26.996 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7280000000000001e-06 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015756+0000", 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5690000000000001e-06 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024305+0000", 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0085489969999999992 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.997 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024312+0000", 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6760000000000001e-06 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063040+0000", 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0387282419999999 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.998 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063151+0000", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111705 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063734+0000", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00058296499999999996 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070400+0000", 2024-09-13T15:08:26.999 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066657829999999998 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070456+0000", 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5537e-05 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077760+0000", 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073039439999999997 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.000 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077809+0000", 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8980000000000002e-05 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077839+0000", 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9844000000000001e-05 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213369 13.5 13:af8813b1:::benchmark_data_smithi042_286876_object89868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:08:27.001 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.87968251199999, 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.043141705, 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213369 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:08:27.002 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034851+0000", 2024-09-13T15:08:27.003 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0219999999999999e-06 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034856+0000", 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.887e-06 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034860+0000", 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4989999999999996e-06 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.004 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037222+0000", 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023623400000000001 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037232+0000", 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.3619999999999995e-06 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066884+0000", 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0296525050000001 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.005 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066958+0000", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3930999999999994e-05 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.067044+0000", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5995999999999995e-05 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070444+0000", 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034000940000000002 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.006 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070463+0000", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.821e-05 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077940+0000", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0074770849999999996 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077970+0000", 2024-09-13T15:08:27.007 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9966000000000001e-05 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077991+0000", 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1169e-05 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213373 13.5 13:aceb1f16:::benchmark_data_smithi042_286876_object89872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.87772744, 2024-09-13T15:08:27.008 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0412968069999999, 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213373 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.009 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036807+0000", 2024-09-13T15:08:27.010 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5159999999999999e-06 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036814+0000", 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2679999999999997e-06 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036820+0000", 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5829999999999997e-06 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.011 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037254+0000", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00043378200000000002 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037259+0000", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8089999999999998e-06 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070497+0000", 2024-09-13T15:08:27.012 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0332373290000001 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070622+0000", 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000124762 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070772+0000", 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015073899999999999 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.013 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077900+0000", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071275740000000002 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077998+0000", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8414999999999999e-05 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078059+0000", 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0547999999999997e-05 2024-09-13T15:08:27.014 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078083+0000", 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3706e-05 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078101+0000", 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.844e-05 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.015 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213380 13.5 13:a8878734:::benchmark_data_smithi042_286876_object89879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.868890042, 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0325521609999999, 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213380 2024-09-13T15:08:27.016 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.017 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045640+0000", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045646+0000", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5490000000000003e-06 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045659+0000", 2024-09-13T15:08:27.018 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3157e-05 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045666+0000", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5740000000000001e-06 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045708+0000", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2543999999999998e-05 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045717+0000", 2024-09-13T15:08:27.019 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6859999999999995e-06 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074126+0000", 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0284085199999999 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074235+0000", 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109467 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.020 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074355+0000", 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000120294 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078004+0000", 2024-09-13T15:08:27.021 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036484629999999998 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078032+0000", 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8059e-05 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.022 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078135+0000", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000102682 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078162+0000", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.743e-05 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078194+0000", 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1788000000000001e-05 2024-09-13T15:08:27.023 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213388 13.5 13:aaa5fe06:::benchmark_data_smithi042_286876_object89887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.854775679, 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0326702839999999, 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.024 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213388 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059755+0000", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059760+0000", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9969999999999998e-06 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059767+0000", 2024-09-13T15:08:27.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2150000000000004e-06 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059772+0000", 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4450000000000004e-06 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059822+0000", 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9412000000000001e-05 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.027 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059831+0000", 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5489999999999995e-06 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078221+0000", 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.018389196 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078296+0000", 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4994999999999996e-05 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.028 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078408+0000", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112861 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.085959+0000", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0075507389999999999 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.085995+0000", 2024-09-13T15:08:27.029 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5725000000000001e-05 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092381+0000", 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0063859390000000002 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092405+0000", 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3830999999999999e-05 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.030 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092426+0000", 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1597e-05 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213390 13.5 13:a04508b5:::benchmark_data_smithi042_286876_object89889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.854439066, 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0323922619999999, 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.031 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213390 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.032 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060092+0000", 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060096+0000", 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3640000000000001e-06 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060105+0000", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7280000000000001e-06 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060110+0000", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1250000000000001e-06 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060142+0000", 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2400999999999997e-05 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060150+0000", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2980000000000003e-06 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.082957+0000", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.022807571 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.083061+0000", 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103991 2024-09-13T15:08:27.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.083139+0000", 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.729e-05 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086032+0000", 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028935480000000001 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092305+0000", 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0062728180000000003 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092433+0000", 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012801699999999999 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092456+0000", 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2623e-05 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092485+0000", 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9295999999999999e-05 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213423 13.5 13:afde02ce:::benchmark_data_smithi042_286876_object89922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.819504779, 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99753285599999997, 2024-09-13T15:08:27.038 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213423 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095025+0000", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095031+0000", 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2669999999999997e-06 2024-09-13T15:08:27.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095040+0000", 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1819999999999995e-06 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095046+0000", 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9209999999999997e-06 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.096636+0000", 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0015904809999999999 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.096644+0000", 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9289999999999998e-06 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086059+0000", 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98941457200000005 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086136+0000", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7562e-05 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086222+0000", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5393e-05 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092250+0000", 2024-09-13T15:08:27.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0060284639999999999 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092374+0000", 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012345800000000001 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092493+0000", 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000119329 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092526+0000", 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3481e-05 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092560+0000", 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3201000000000003e-05 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.045 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213445 13.5 13:aa84eeca:::benchmark_data_smithi042_286876_object89944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.798682633, 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98705722900000004, 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213445 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.046 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115848+0000", 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115852+0000", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3340000000000003e-06 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115862+0000", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6600000000000007e-06 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115869+0000", 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4300000000000003e-06 2024-09-13T15:08:27.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.116666+0000", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00079784300000000001 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.116674+0000", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8399999999999995e-06 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089371+0000", 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97269705500000003 2024-09-13T15:08:27.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089445+0000", 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3355000000000005e-05 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089530+0000", 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5198999999999995e-05 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092344+0000", 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002814263 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.096027+0000", 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036830320000000001 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102852+0000", 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0068251869999999999 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102884+0000", 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1445000000000001e-05 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102906+0000", 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2470999999999999e-05 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213452 13.5 13:af44a94b:::benchmark_data_smithi042_286876_object89951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.052 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.790246701, 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97874794499999995, 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213452 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124284+0000", 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:27.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124288+0000", 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5460000000000002e-06 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124299+0000", 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0478e-05 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124304+0000", 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7899999999999999e-06 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124339+0000", 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5309999999999999e-05 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124349+0000", 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9590000000000007e-06 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092591+0000", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96824188899999997 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092671+0000", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9610000000000005e-05 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092793+0000", 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012206600000000001 2024-09-13T15:08:27.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.096058+0000", 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032655029999999999 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102845+0000", 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067864350000000004 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102979+0000", 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013441700000000001 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.103007+0000", 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8161999999999999e-05 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.103033+0000", 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5748000000000001e-05 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.059 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213538 13.5 13:a51335ba:::benchmark_data_smithi042_286876_object90037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 253.674337291, 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.88139930200000005, 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213538 2024-09-13T15:08:27.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240198+0000", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.566e-06 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240212+0000", 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4149e-05 2024-09-13T15:08:27.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240219+0000", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3999999999999997e-06 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240262+0000", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3509e-05 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240271+0000", 2024-09-13T15:08:27.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7960000000000001e-06 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099494+0000", 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85922312099999998 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099633+0000", 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013862799999999999 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:08:27.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099753+0000", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012005100000000001 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102972+0000", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032194789999999999 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.117182+0000", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.014209816 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:08:27.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121539+0000", 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043566949999999998 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121568+0000", 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9218e-05 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121594+0000", 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.565e-05 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.066 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.067 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.067 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:27.067 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:27.067 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:08:27.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.069 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:27.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:27.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:27.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.214 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:27.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:27.287 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:27.315 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:27.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.374 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:27.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:27.476 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:27.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:27.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.516 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:27.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:27.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:27.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:27.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:27.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:08:27.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:08:27.600 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:27.617 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:27.649 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:27.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:27.751 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.782 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:27.782 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:27.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:27.916 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:27.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:27.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:27.999 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:28.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:28.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:28.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:28.069 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:08:28.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:28.205 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:28.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:28.307 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.343 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:28.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:28.409 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:28.443 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:28.480 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:28.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:28.582 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:28.596 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:28 smithi042 ceph-mon[108570]: pgmap v1554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:28.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:28 smithi042 ceph-mon[113926]: pgmap v1554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:28.631 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:28.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:28.717 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:28 smithi159 ceph-mon[102519]: pgmap v1554: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 963 op/s; 2730/159615 objects degraded (1.710%); 9254/159615 objects misplaced (5.798%); 1.1 MiB/s, 16 objects/s recovering 2024-09-13T15:08:28.733 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:28.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.770 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:28.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:28.871 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:28.907 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:28.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:29.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:29.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:29.154 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:29.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:29.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:29.323 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:29.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:29.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:29.460 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:29.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:29.562 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.592 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:29.694 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:29.728 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:29.830 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:29.850 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:29] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-09-13T15:08:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:29.864 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:29.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:29.965 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:30.006 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:30.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:30.108 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:30.145 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:30.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:30.247 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:30.284 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:30.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:30.386 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:30.425 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:30.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:30.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.572 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:30.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:30.673 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:30.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:30.745 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:30 smithi042 ceph-mon[108570]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 909 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:30.745 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:30 smithi042 ceph-mon[113926]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 909 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:30.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:30 smithi159 ceph-mon[102519]: pgmap v1555: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 909 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:30.808 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:30.842 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:30.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:30.943 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:30.975 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:30.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:31.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:31.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:31.219 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.254 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:31.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:31.356 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:31.385 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:31.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:31.487 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:31.520 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:31.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:31.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:31 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2914/146457 objects degraded (1.990%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:31.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:31 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2914/146457 objects degraded (1.990%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:31.622 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:31.664 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:31.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:31 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2914/146457 objects degraded (1.990%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:31.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.807 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:31.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:31.908 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:31.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:31.947 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:31.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:32.048 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:32.088 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:32.192 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.222 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:32.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:32.324 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.360 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:32.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:32.405 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:08:32.462 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:32.518 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:32 smithi159 ceph-mon[102519]: pgmap v1556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:08:32.518 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3277309531' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.518 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:32 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.566 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:32.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:32.586 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:08:32.586 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:08:32.586 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:203958 14.3 e434/416 14:cb69b2cb:::benchmark_data_smithi042_319156_object80001:head v 434'12994, mlcod=434'12991)", 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:32.580833+0000", 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0036990759999999999, 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037283149999999998, 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.580833+0000", 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.587 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.580833+0000", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.580832+0000", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.580840+0000", 2024-09-13T15:08:32.588 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.259e-06 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.580872+0000", 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2123999999999999e-05 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581739+0000", 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00086656700000000005 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.589 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581967+0000", 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000228617 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.582023+0000", 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5569000000000003e-05 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.590 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:203959 14.3 e434/416)", 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:32.581760+0000", 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.0027717000000000002, 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028609019999999998, 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "queued for pg", 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581760+0000", 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:32.591 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581760+0000", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581760+0000", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581765+0000", 2024-09-13T15:08:32.592 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2719999999999997e-06 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581772+0000", 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4209999999999998e-06 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:32.581778+0000", 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6210000000000001e-06 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:32.593 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:32.594 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T15:08:32.594 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 2 2024-09-13T15:08:32.594 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:08:32.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:32.711 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:32.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:32.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[113926]: pgmap v1556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:08:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3277309531' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[108570]: pgmap v1556: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 3.0 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 2914/146457 objects degraded (1.990%); 8452/146457 objects misplaced (5.771%); 1.0 MiB/s, 16 objects/s recovering 2024-09-13T15:08:32.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3277309531' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:32 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:08:32.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.859 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:32.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:32.960 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:32.995 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:32.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:33.097 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.128 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:33.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:33.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:33.264 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:33.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:33.329 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:08:33.366 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:33.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:33.510 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.554 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:33.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:33.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:08:33.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[113926]: osdmap e434: 8 total, 8 up, 7 in 2024-09-13T15:08:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:08:33.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:33 smithi042 ceph-mon[108570]: osdmap e434: 8 total, 8 up, 7 in 2024-09-13T15:08:33.655 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:33.683 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:33.685 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:33 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:33.685 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:33 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:08:33.685 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:33 smithi159 ceph-mon[102519]: osdmap e434: 8 total, 8 up, 7 in 2024-09-13T15:08:33.686 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:33.785 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.829 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:33.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:33.930 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:33.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:33.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:33.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:34.067 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.100 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:34.159 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:08:34.159 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:34.188 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:08:34.188 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:34.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:34.221 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:08:34.221 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.240 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:34.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:34.257 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:08:34.257 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:34.310 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:08:34.311 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:34.341 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:34.349 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:08:34.350 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:34.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:34.391 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:08:34.391 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:34.431 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:08:34.432 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:34.491 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:34.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.528 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:34.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:34.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:34 smithi159 ceph-mon[102519]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2914/135927 objects degraded (2.144%); 8179/135927 objects misplaced (6.017%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:34.623 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:34 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1671866170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:34.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:34.668 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:34.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:34.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:34.809 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:34 smithi042 ceph-mon[108570]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2914/135927 objects degraded (2.144%); 8179/135927 objects misplaced (6.017%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:34.810 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:34 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1671866170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:34.810 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:34 smithi042 ceph-mon[113926]: pgmap v1558: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 905 op/s; 2914/135927 objects degraded (2.144%); 8179/135927 objects misplaced (6.017%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:34.810 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:34 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1671866170' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.811 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:34.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:34.913 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:34.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:34.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:34.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:35.053 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:35.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:35.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:35.198 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:35.246 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:35.348 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:35.384 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:35.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:35.446 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 14 pg_num to 26 2024-09-13T15:08:35.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:35.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:35.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:35.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:35 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2953997896' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:35 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:35 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2953997896' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:35 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.641 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:35 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2953997896' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.641 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:35 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"} : dispatch 2024-09-13T15:08:35.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:35.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:35.765 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:35.806 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:35.806 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:35.807 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:35.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:35.908 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:35.915 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-13T15:08:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:35.943 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:35.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:36.045 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:36.075 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:36.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:36.177 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:36.216 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:36.318 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:36.352 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:36.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:36.454 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:36.484 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:36.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:36.512 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:08:36.587 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:36.623 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:36.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:36.726 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: pgmap v1559: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: osdmap e435: 8 total, 8 up, 7 in 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 3119/132348 objects degraded (2.357%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:36.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:36.754 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:36.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:36.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: pgmap v1559: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: osdmap e435: 8 total, 8 up, 7 in 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 3119/132348 objects degraded (2.357%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: pgmap v1559: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 908 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 1.1 MiB/s, 17 objects/s recovering 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "26"}]': finished 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: osdmap e435: 8 total, 8 up, 7 in 2024-09-13T15:08:36.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 3119/132348 objects degraded (2.357%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-09-13T15:08:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:08:36.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:36.856 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:36.903 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:36.907 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:36.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:36.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:37.009 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:37.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:37.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:37.153 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:37.188 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:37.290 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:37.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:08:37.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.345 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:37.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:37.446 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:37.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:37.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.601 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:37 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:08:37.601 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:37 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:08:37.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.666 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:37.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:37.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:37 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-13T15:08:37.768 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:37.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.816 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:37.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:37.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:37.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:37.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:37.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:38.041 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:38.063 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:38.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:38.164 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:38.186 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:38.288 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:38.310 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:38.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:38.412 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:38.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:38.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:38.535 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:38.563 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:38.665 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:38.692 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:38 smithi159 ceph-mon[102519]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 752 KiB/s, 11 objects/s recovering 2024-09-13T15:08:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:38 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:08:38.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:38 smithi159 ceph-mon[102519]: osdmap e436: 8 total, 8 up, 7 in 2024-09-13T15:08:38.784 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[113926]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 752 KiB/s, 11 objects/s recovering 2024-09-13T15:08:38.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:08:38.785 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[113926]: osdmap e436: 8 total, 8 up, 7 in 2024-09-13T15:08:38.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[108570]: pgmap v1561: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 95 active+clean; 2.7 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 587 op/s; 3119/132348 objects degraded (2.357%); 7590/132348 objects misplaced (5.735%); 752 KiB/s, 11 objects/s recovering 2024-09-13T15:08:38.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-13T15:08:38.785 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:38 smithi042 ceph-mon[108570]: osdmap e436: 8 total, 8 up, 7 in 2024-09-13T15:08:38.793 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:38.817 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:38.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:38.943 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:38.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:39.044 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:39.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.068 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:39.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:39.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:39.194 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:39.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:39.296 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:39.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:39.423 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:39.446 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:39.542 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:39.542 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:39.543 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:39 smithi159 ceph-mon[102519]: 14.8 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:39.543 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:39 smithi159 ceph-mon[102519]: osdmap e437: 8 total, 8 up, 7 in 2024-09-13T15:08:39.543 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:39 smithi159 ceph-mon[102519]: 14.2 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:39.543 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:39 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:08:39.543 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:39 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:08:39.547 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:39.570 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:39.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:39.672 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:39.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:39.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:39.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:39.822 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:39.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:39] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-13T15:08:39.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[113926]: 14.8 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[113926]: osdmap e437: 8 total, 8 up, 7 in 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[113926]: 14.2 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[108570]: 14.8 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[108570]: osdmap e437: 8 total, 8 up, 7 in 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[108570]: 14.2 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:08:39.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:39 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:08:39.924 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:39.948 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:39.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:40.050 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:40.073 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:40.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:40.175 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:40.200 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:40.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:40.208 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:08:40.302 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:40.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:40.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:40.427 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:40.438 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:40.449 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:40.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:40.551 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:40.585 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:40.687 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:40.718 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:40.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:40.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:40 smithi159 ceph-mon[102519]: pgmap v1564: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:40.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:40 smithi159 ceph-mon[102519]: osdmap e438: 8 total, 8 up, 7 in 2024-09-13T15:08:40.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:40 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:08:40.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:40 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:08:40.821 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:40.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[108570]: pgmap v1564: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[108570]: osdmap e438: 8 total, 8 up, 7 in 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[113926]: pgmap v1564: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.8 MiB/s, 28 objects/s recovering 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[113926]: osdmap e438: 8 total, 8 up, 7 in 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:08:40.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:40 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:08:40.855 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:40.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:40.957 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:40.993 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:40.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:40.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:41.095 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:41.130 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:41.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:41.232 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:41.274 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:41.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:41.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:41.399 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:41.501 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:41.533 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:41.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:41.635 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:41.666 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:41.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:41.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:41 smithi159 ceph-mon[102519]: 14.18 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:41.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:41 smithi159 ceph-mon[102519]: osdmap e439: 8 total, 8 up, 7 in 2024-09-13T15:08:41.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:41 smithi159 ceph-mon[102519]: 14.12 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:41.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:41 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:08:41.711 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:41 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:08:41.767 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:41.800 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:41.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[108570]: 14.18 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[108570]: osdmap e439: 8 total, 8 up, 7 in 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[108570]: 14.12 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[113926]: 14.18 continuing backfill to osd.5 from (404'5700,435'12928] MIN to 435'12928 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[113926]: osdmap e439: 8 total, 8 up, 7 in 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[113926]: 14.12 continuing backfill to osd.5 from (404'6505,435'13118] MIN to 435'13118 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:08:41.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:41 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:08:41.903 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:41.940 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:41.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:41.993 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:42.021 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:42.042 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.067 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.169 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.193 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:42.306 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:42.307 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:42.321 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:42.423 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:42.446 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:42.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:42.548 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:42.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:42.685 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:42.722 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:42.743 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:42 smithi159 ceph-mon[102519]: pgmap v1567: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:08:42.743 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:42 smithi159 ceph-mon[102519]: osdmap e440: 8 total, 8 up, 7 in 2024-09-13T15:08:42.743 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:42 smithi159 ceph-mon[102519]: 14.11 scrub starts 2024-09-13T15:08:42.825 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[113926]: pgmap v1567: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:08:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[113926]: osdmap e440: 8 total, 8 up, 7 in 2024-09-13T15:08:42.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[113926]: 14.11 scrub starts 2024-09-13T15:08:42.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[108570]: pgmap v1567: 109 pgs: 4 peering, 2 remapped+peering, 10 unknown, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.11k op/s; 1/119046 objects degraded (0.001%); 2497/119046 objects misplaced (2.098%); 1.2 MiB/s, 19 objects/s recovering 2024-09-13T15:08:42.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[108570]: osdmap e440: 8 total, 8 up, 7 in 2024-09-13T15:08:42.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:42 smithi042 ceph-mon[108570]: 14.11 scrub starts 2024-09-13T15:08:42.857 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:42.958 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:42.990 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:42.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:43.092 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:43.101 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:08:43.129 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:43.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:43.231 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:43.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.270 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:43.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:43.371 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:43.404 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:43.470 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:43.505 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:43.543 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:43.645 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:43.684 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:43.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:43.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:43.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:43.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:43 smithi042 ceph-mon[113926]: osdmap e441: 8 total, 8 up, 7 in 2024-09-13T15:08:43.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:43 smithi042 ceph-mon[113926]: 14.19 scrub starts 2024-09-13T15:08:43.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:43 smithi042 ceph-mon[108570]: osdmap e441: 8 total, 8 up, 7 in 2024-09-13T15:08:43.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:43 smithi042 ceph-mon[108570]: 14.19 scrub starts 2024-09-13T15:08:43.920 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:43.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:43.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:43.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:43 smithi159 ceph-mon[102519]: osdmap e441: 8 total, 8 up, 7 in 2024-09-13T15:08:43.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:43 smithi159 ceph-mon[102519]: 14.19 scrub starts 2024-09-13T15:08:44.052 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:44.092 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:44.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:44.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:44.218 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:44.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:44.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:44.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:44.456 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:44.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:44.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:44.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:44.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:44.766 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:44.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[113926]: pgmap v1570: 109 pgs: 1 active+remapped+backfill_wait, 5 peering, 2 remapped+peering, 6 unknown, 95 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 1179/115914 objects misplaced (1.017%); 19 MiB/s, 309 objects/s recovering 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded) 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[108570]: pgmap v1570: 109 pgs: 1 active+remapped+backfill_wait, 5 peering, 2 remapped+peering, 6 unknown, 95 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 1179/115914 objects misplaced (1.017%); 19 MiB/s, 309 objects/s recovering 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded) 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:08:44.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:44 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:08:44.868 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:44.904 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:44.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:44.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:44 smithi159 ceph-mon[102519]: pgmap v1570: 109 pgs: 1 active+remapped+backfill_wait, 5 peering, 2 remapped+peering, 6 unknown, 95 active+clean; 2.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 260 op/s; 1179/115914 objects misplaced (1.017%); 19 MiB/s, 309 objects/s recovering 2024-09-13T15:08:44.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:44 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/119046 objects degraded (0.001%), 1 pg degraded) 2024-09-13T15:08:44.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:44 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:08:44.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:44 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:08:45.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.042 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.144 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:45.175 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:45.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:45.308 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:45.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:45.410 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.543 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:45.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:45.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:45.705 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:45.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:45.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:45.833 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:45.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:45 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:08:45.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:45 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:08:45.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:45 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:08:45.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:45 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:08:45.934 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:45.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:45.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:45 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:08:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:45 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:08:46.071 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:46.103 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:46.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:46.206 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.231 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:46.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:46.333 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:46.386 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:46.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:46.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:46.520 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:46.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:46.622 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:46.660 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:46.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:46 smithi159 ceph-mon[102519]: pgmap v1571: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 30 MiB/s, 479 objects/s recovering 2024-09-13T15:08:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:46 smithi159 ceph-mon[102519]: Health check failed: Degraded data redundancy: 2547/112083 objects degraded (2.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:46.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:46 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:08:46.762 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:46.791 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:46.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[113926]: pgmap v1571: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 30 MiB/s, 479 objects/s recovering 2024-09-13T15:08:46.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[113926]: Health check failed: Degraded data redundancy: 2547/112083 objects degraded (2.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:46.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:08:46.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[108570]: pgmap v1571: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 30 MiB/s, 479 objects/s recovering 2024-09-13T15:08:46.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[108570]: Health check failed: Degraded data redundancy: 2547/112083 objects degraded (2.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:46.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:46 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:08:46.894 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:46.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:46.924 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:46.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:46.948 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:08:46.948 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:08:46.948 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_9 2024-09-13T15:08:46.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:08:47.025 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:47.052 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:47.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:47.154 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:47.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.182 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:47.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:47.284 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:47.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:47.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:47.410 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:47.432 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:47.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:47.534 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:47.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:47.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:47.706 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:47.798 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:47.798 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:47.799 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:47 smithi159 ceph-mon[102519]: osdmap e442: 8 total, 8 up, 7 in 2024-09-13T15:08:47.799 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:47 smithi159 ceph-mon[102519]: pgmap v1573: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 26 MiB/s, 423 objects/s recovering 2024-09-13T15:08:47.799 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:47 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:08:47.808 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:47.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[108570]: osdmap e442: 8 total, 8 up, 7 in 2024-09-13T15:08:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[108570]: pgmap v1573: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 26 MiB/s, 423 objects/s recovering 2024-09-13T15:08:47.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:08:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[113926]: osdmap e442: 8 total, 8 up, 7 in 2024-09-13T15:08:47.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[113926]: pgmap v1573: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 peering, 103 active+clean; 2.3 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 386 op/s; 2547/112083 objects degraded (2.272%); 3087/112083 objects misplaced (2.754%); 26 MiB/s, 423 objects/s recovering 2024-09-13T15:08:47.850 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:47 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:08:47.941 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:47.974 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:47.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:47.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:48.076 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:48.112 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:48.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:48.214 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:48.250 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:48.352 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:48.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.391 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:48.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:48.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:48.526 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:08:48.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.527 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:48.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:48.628 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:48.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:48.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:48 smithi159 ceph-mon[102519]: osdmap e443: 8 total, 8 up, 7 in 2024-09-13T15:08:48.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:48 smithi159 ceph-mon[102519]: 14.8 scrub starts 2024-09-13T15:08:48.750 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:48 smithi159 ceph-mon[102519]: 14.11 scrub starts 2024-09-13T15:08:48.770 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:48.816 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:48.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:48.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[108570]: osdmap e443: 8 total, 8 up, 7 in 2024-09-13T15:08:48.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[108570]: 14.8 scrub starts 2024-09-13T15:08:48.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[108570]: 14.11 scrub starts 2024-09-13T15:08:48.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[113926]: osdmap e443: 8 total, 8 up, 7 in 2024-09-13T15:08:48.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[113926]: 14.8 scrub starts 2024-09-13T15:08:48.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:48 smithi042 ceph-mon[113926]: 14.11 scrub starts 2024-09-13T15:08:48.861 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:48.918 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:48.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:48.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:48.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:49.058 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.097 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:49.198 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:49.233 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:49.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:49.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.371 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:49.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:49.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:49.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:49.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:49.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:49 smithi159 ceph-mon[102519]: pgmap v1575: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 22 MiB/s, 376 objects/s recovering 2024-09-13T15:08:49.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:49 smithi159 ceph-mon[102519]: 14.19 scrub starts 2024-09-13T15:08:49.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.779 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:49.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:49.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:49 smithi042 ceph-mon[108570]: pgmap v1575: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 22 MiB/s, 376 objects/s recovering 2024-09-13T15:08:49.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:49 smithi042 ceph-mon[108570]: 14.19 scrub starts 2024-09-13T15:08:49.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:49 smithi042 ceph-mon[113926]: pgmap v1575: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.12k op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 22 MiB/s, 376 objects/s recovering 2024-09-13T15:08:49.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:49 smithi042 ceph-mon[113926]: 14.19 scrub starts 2024-09-13T15:08:49.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:49] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-09-13T15:08:49.881 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:49.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:49.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:49.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:50.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:50.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:50.160 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:50.193 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:50.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:50.295 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:50.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.333 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:50.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:50.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:50.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:50.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:50.574 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:50.610 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:50.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:50.744 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:50.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:50 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:08:50.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:50 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:08:50.757 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:50 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:08:50.757 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:50 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:08:50.758 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:50 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:08:50.758 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:50 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:08:50.836 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:50.846 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:50.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:50.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:50.981 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:51.018 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:51.113 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:08:51.113 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:08:51.120 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:51.161 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:51.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:51.263 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:51.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:51.397 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:51.434 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:51.536 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.572 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:51.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:51.673 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:51.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.709 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:51.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2546/94749 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[108570]: pgmap v1576: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 17 MiB/s, 295 objects/s recovering 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2546/94749 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:51.718 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[113926]: pgmap v1576: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 17 MiB/s, 295 objects/s recovering 2024-09-13T15:08:51.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:51.719 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:51 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:08:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:51 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:08:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:51 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2546/94749 objects degraded (2.687%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:51 smithi159 ceph-mon[102519]: pgmap v1576: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 881 op/s; 2546/94749 objects degraded (2.687%); 1743/94749 objects misplaced (1.840%); 17 MiB/s, 295 objects/s recovering 2024-09-13T15:08:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:08:51.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:51 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:08:51.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:51.844 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:51.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:51.867 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:08:51.874 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:08:51.874 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:08:51.901 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:08:51.902 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:08:51.926 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:08:51.926 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:08:51.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:51.950 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:08:51.951 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:08:51.978 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:51.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:51.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:08:51.981 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:08:52.007 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:08:52.008 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:08:52.029 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:08:52.030 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:08:52.051 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:08:52.051 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:08:52.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:52.170 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:52.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:52.232 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:52.334 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:52.357 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:52.459 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.481 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:52.583 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:52.586 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is set 2024-09-13T15:08:52.612 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:52.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:52 smithi159 ceph-mon[102519]: osdmap e444: 8 total, 8 up, 7 in 2024-09-13T15:08:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3639014977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2816819955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:52 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:52 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:08:52.756 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:52.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[113926]: osdmap e444: 8 total, 8 up, 7 in 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3639014977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2816819955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[108570]: osdmap e444: 8 total, 8 up, 7 in 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3639014977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:08:52.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2816819955' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:08:52.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:52 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:08:52.858 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:52.890 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:52.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:52.991 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:53.032 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:53.134 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:53.174 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:53.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:53.307 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:53.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:53.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:53.458 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:53.560 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:53.610 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:53.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.745 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:53.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:53.789 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[113926]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[113926]: osdmap e445: 8 total, 8 up, 7 in 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[113926]: pgmap v1579: 109 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/86223 objects degraded (2.953%); 945/86223 objects misplaced (1.096%); 17 MiB/s, 302 objects/s recovering 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[108570]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[108570]: osdmap e445: 8 total, 8 up, 7 in 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[108570]: pgmap v1579: 109 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/86223 objects degraded (2.953%); 945/86223 objects misplaced (1.096%); 17 MiB/s, 302 objects/s recovering 2024-09-13T15:08:53.790 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:53 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:08:53.847 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:53.881 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:53.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:53.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:53.891 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:53 smithi159 ceph-mon[102519]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:08:53.891 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:53 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-13T15:08:53.891 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:53 smithi159 ceph-mon[102519]: osdmap e445: 8 total, 8 up, 7 in 2024-09-13T15:08:53.892 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:53 smithi159 ceph-mon[102519]: pgmap v1579: 109 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/86223 objects degraded (2.953%); 945/86223 objects misplaced (1.096%); 17 MiB/s, 302 objects/s recovering 2024-09-13T15:08:53.892 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:53 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:08:53.982 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:54.021 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:54.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:54.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:54.151 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:223816 14.f e445/422 14:f9765a11:::benchmark_data_smithi042_319156_object99859:head v 445'13975, mlcod=445'13973)", 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:54.228805+0000", 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 0.00070464400000000004, 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00073802700000000004, 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "started", 2024-09-13T15:08:54.232 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228805+0000", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228805+0000", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228804+0000", 2024-09-13T15:08:54.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228811+0000", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1810000000000004e-06 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228826+0000", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5415000000000001e-05 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228836+0000", 2024-09-13T15:08:54.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5710000000000004e-06 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228888+0000", 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2370000000000002e-05 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:54.228920+0000", 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1568e-05 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:08:54.235 INFO:teuthology.orchestra.run.smithi159.stdout: ], 2024-09-13T15:08:54.236 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 1 2024-09-13T15:08:54.236 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:08:54.253 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:54.287 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:54.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:54.420 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:54.522 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:54.558 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:54.660 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:54.798 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:54.838 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:54.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:54 smithi042 ceph-mon[113926]: 14.8 scrub starts 2024-09-13T15:08:54.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:54 smithi042 ceph-mon[113926]: 14.11 scrub starts 2024-09-13T15:08:54.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:54 smithi042 ceph-mon[108570]: 14.8 scrub starts 2024-09-13T15:08:54.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:54 smithi042 ceph-mon[108570]: 14.11 scrub starts 2024-09-13T15:08:54.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:54.962 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:54.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:54.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:54 smithi159 ceph-mon[102519]: 14.8 scrub starts 2024-09-13T15:08:54.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:54 smithi159 ceph-mon[102519]: 14.11 scrub starts 2024-09-13T15:08:55.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:08:55.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:55.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.125 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:55.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:55.226 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:55.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:55.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:55.370 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:55.388 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:55.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:55.437 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:08:55.490 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:55.519 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:55.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:55.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:55.661 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[113926]: pgmap v1580: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 13 MiB/s, 290 objects/s recovering 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[113926]: 14.19 scrub starts 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[108570]: pgmap v1580: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 13 MiB/s, 290 objects/s recovering 2024-09-13T15:08:55.750 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:55 smithi042 ceph-mon[108570]: 14.19 scrub starts 2024-09-13T15:08:55.764 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.805 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:55.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:55.872 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:55 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:08:55.872 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:55 smithi159 ceph-mon[102519]: pgmap v1580: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 13 MiB/s, 290 objects/s recovering 2024-09-13T15:08:55.872 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:55 smithi159 ceph-mon[102519]: 14.19 scrub starts 2024-09-13T15:08:55.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:55.939 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:55.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:55.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:56.041 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:56.080 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:56.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:56.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.222 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:56.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:56.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:08:56.356 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:56.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:08:56.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:56.492 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:56.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:56.595 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:56.627 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:56.698 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:56 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:08:56.698 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:56 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2546/77820 objects degraded (3.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:56.698 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:56 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:08:56.729 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:56.782 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:56.884 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:56.914 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:56.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:56.953 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:08:56.953 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2546/77820 objects degraded (3.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:56.953 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:08:56.953 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:08:56.954 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2546/77820 objects degraded (3.272%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:08:56.954 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:56 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:08:57.016 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.058 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:57.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:57.159 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:57.208 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:57.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:57.310 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:57.355 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:57.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:57.405 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:08:57.405 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:08:57.405 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:08:57.405 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:08:57.405 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:08:57.457 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:08:57.473 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:08:57.487 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:57.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:08:57.588 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:57.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:57.744 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:57.781 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:57.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:57.885 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:08:57.887 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:57 smithi042 ceph-mon[113926]: pgmap v1581: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 5.8 MiB/s, 154 objects/s recovering 2024-09-13T15:08:57.887 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:57 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:08:57.889 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:08:57.891 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:57 smithi042 ceph-mon[108570]: pgmap v1581: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 5.8 MiB/s, 154 objects/s recovering 2024-09-13T15:08:57.891 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:57 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:08:57.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:57.922 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:57.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:57 smithi159 ceph-mon[102519]: pgmap v1581: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 105 active+clean; 1.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 2546/77820 objects degraded (3.272%); 775/77820 objects misplaced (0.996%); 5.8 MiB/s, 154 objects/s recovering 2024-09-13T15:08:57.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:57 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:08:58.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.068 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:58.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:58.137 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:08:58.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:08:58.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.219 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:58.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:58.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:08:58.365 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:58.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:08:58.467 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.485 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:08:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.506 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:58.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:58.606 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.651 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:58.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:58.753 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:58.790 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:58.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:58 smithi042 ceph-mon[108570]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:58.823 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:58 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:08:58.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:58 smithi042 ceph-mon[113926]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:58.824 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:58 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:08:58.892 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:58.927 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:58.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:58.960 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:58 smithi159 ceph-mon[102519]: from='client.46307 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:08:58.960 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:58 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:08:59.029 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:08:59.064 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:08:59.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:59.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:59.304 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:08:59.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.341 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:59.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:08:59.442 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:08:59.482 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:08:59.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:59.616 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:59.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:59.719 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:08:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:59 smithi159 ceph-mon[102519]: 14.8 scrub starts 2024-09-13T15:08:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:59 smithi159 ceph-mon[102519]: pgmap v1582: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.8 MiB/s, 197 objects/s recovering 2024-09-13T15:08:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:08:59 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:08:59.752 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:08:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:08:59.854 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:08:59.873 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1581,"stamp":"2024-09-13T15:08:56.785116+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1698830333,"num_objects":25940,"num_object_clones":0,"num_object_copies":77820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":775,"num_objects_unfound":0,"num_objects_dirty":25940,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":222060,"num_write_kb":7933090,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21589,"num_bytes_recovered":1368397507,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176524,"ondisk_log_size":176524,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15132712,"kb_used_data":5774696,"kb_used_omap":22,"kb_used_meta":9357929,"kb_avail":641083352,"statfs":{"total":671965249536,"available":656469352448,"internally_reserved":0,"allocated":5913288704,"data_stored":5896726924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23321,"internal_metadata":9582519527},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-369819648,"num_objects":-5643,"num_object_clones":0,"num_object_copies":-16929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-968,"num_objects_unfound":0,"num_objects_dirty":-5643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1239,"num_bytes_recovered":48955392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002266"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":726,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584206+0000","last_change":"2024-09-13T15:06:40.609190+0000","last_active":"2024-09-13T15:08:52.584206+0000","last_peered":"2024-09-13T15:08:52.584206+0000","last_clean":"2024-09-13T15:08:52.584206+0000","last_became_active":"2024-09-13T15:06:40.608791+0000","last_became_peered":"2024-09-13T15:06:40.608791+0000","last_unstale":"2024-09-13T15:08:52.584206+0000","last_undegraded":"2024-09-13T15:08:52.584206+0000","last_fullsized":"2024-09-13T15:08:52.584206+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:19.550059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":139,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578357+0000","last_change":"2024-09-13T14:59:12.723780+0000","last_active":"2024-09-13T15:08:47.578357+0000","last_peered":"2024-09-13T15:08:47.578357+0000","last_clean":"2024-09-13T15:08:47.578357+0000","last_became_active":"2024-09-13T14:59:12.707778+0000","last_became_peered":"2024-09-13T14:59:12.707778+0000","last_unstale":"2024-09-13T15:08:47.578357+0000","last_undegraded":"2024-09-13T15:08:47.578357+0000","last_fullsized":"2024-09-13T15:08:47.578357+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:15:55.159858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583801+0000","last_change":"2024-09-13T15:01:33.854931+0000","last_active":"2024-09-13T15:08:52.583801+0000","last_peered":"2024-09-13T15:08:52.583801+0000","last_clean":"2024-09-13T15:08:52.583801+0000","last_became_active":"2024-09-13T15:01:33.854486+0000","last_became_peered":"2024-09-13T15:01:33.854486+0000","last_unstale":"2024-09-13T15:08:52.583801+0000","last_undegraded":"2024-09-13T15:08:52.583801+0000","last_fullsized":"2024-09-13T15:08:52.583801+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:19.664954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":495,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583165+0000","last_change":"2024-09-13T15:06:40.607992+0000","last_active":"2024-09-13T15:08:52.583165+0000","last_peered":"2024-09-13T15:08:52.583165+0000","last_clean":"2024-09-13T15:08:52.583165+0000","last_became_active":"2024-09-13T15:06:40.607427+0000","last_became_peered":"2024-09-13T15:06:40.607427+0000","last_unstale":"2024-09-13T15:08:52.583165+0000","last_undegraded":"2024-09-13T15:08:52.583165+0000","last_fullsized":"2024-09-13T15:08:52.583165+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:12:57.514127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":489,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584021+0000","last_change":"2024-09-13T15:03:44.128445+0000","last_active":"2024-09-13T15:08:52.584021+0000","last_peered":"2024-09-13T15:08:52.584021+0000","last_clean":"2024-09-13T15:08:52.584021+0000","last_became_active":"2024-09-13T15:03:44.128047+0000","last_became_peered":"2024-09-13T15:03:44.128047+0000","last_unstale":"2024-09-13T15:08:52.584021+0000","last_undegraded":"2024-09-13T15:08:52.584021+0000","last_fullsized":"2024-09-13T15:08:52.584021+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:34:22.359631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":133,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577632+0000","last_change":"2024-09-13T14:59:12.709377+0000","last_active":"2024-09-13T15:08:47.577632+0000","last_peered":"2024-09-13T15:08:47.577632+0000","last_clean":"2024-09-13T15:08:47.577632+0000","last_became_active":"2024-09-13T14:59:12.709006+0000","last_became_peered":"2024-09-13T14:59:12.709006+0000","last_unstale":"2024-09-13T15:08:47.577632+0000","last_undegraded":"2024-09-13T15:08:47.577632+0000","last_fullsized":"2024-09-13T15:08:47.577632+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:42:33.526948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":751,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583162+0000","last_change":"2024-09-13T15:07:52.945830+0000","last_active":"2024-09-13T15:08:52.583162+0000","last_peered":"2024-09-13T15:08:52.583162+0000","last_clean":"2024-09-13T15:08:52.583162+0000","last_became_active":"2024-09-13T15:07:52.945465+0000","last_became_peered":"2024-09-13T15:07:52.945465+0000","last_unstale":"2024-09-13T15:08:52.583162+0000","last_undegraded":"2024-09-13T15:08:52.583162+0000","last_fullsized":"2024-09-13T15:08:52.583162+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:37:15.475946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":538,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555659+0000","last_change":"2024-09-13T15:03:44.130129+0000","last_active":"2024-09-13T15:08:47.555659+0000","last_peered":"2024-09-13T15:08:47.555659+0000","last_clean":"2024-09-13T15:08:47.555659+0000","last_became_active":"2024-09-13T15:03:44.114883+0000","last_became_peered":"2024-09-13T15:03:44.114883+0000","last_unstale":"2024-09-13T15:08:47.555659+0000","last_undegraded":"2024-09-13T15:08:47.555659+0000","last_fullsized":"2024-09-13T15:08:47.555659+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:45:54.544220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.2","version":"444'13525","reported_seq":28208,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.969883+0000","last_change":"2024-09-13T15:08:42.525967+0000","last_active":"2024-09-13T15:08:51.969883+0000","last_peered":"2024-09-13T15:08:51.969883+0000","last_clean":"2024-09-13T15:08:51.969883+0000","last_became_active":"2024-09-13T15:08:42.505349+0000","last_became_peered":"2024-09-13T15:08:42.505349+0000","last_unstale":"2024-09-13T15:08:51.969883+0000","last_undegraded":"2024-09-13T15:08:51.969883+0000","last_fullsized":"2024-09-13T15:08:51.969883+0000","mapping_epoch":440,"log_start":"404'6505","ondisk_log_start":"404'6505","created":395,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7020,"log_dups_size":0,"ondisk_log_size":7020,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53870592,"num_objects":822,"num_object_clones":0,"num_object_copies":2466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6966,"num_write_kb":249216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4257,"num_bytes_recovered":278888448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":174,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.579414+0000","last_change":"2024-09-13T15:06:40.604087+0000","last_active":"2024-09-13T15:08:47.579414+0000","last_peered":"2024-09-13T15:08:47.579414+0000","last_clean":"2024-09-13T15:08:47.579414+0000","last_became_active":"2024-09-13T15:06:40.600583+0000","last_became_peered":"2024-09-13T15:06:40.600583+0000","last_unstale":"2024-09-13T15:08:47.579414+0000","last_undegraded":"2024-09-13T15:08:47.579414+0000","last_fullsized":"2024-09-13T15:08:47.579414+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:08:09.743062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555567+0000","last_change":"2024-09-13T15:01:33.852319+0000","last_active":"2024-09-13T15:08:47.555567+0000","last_peered":"2024-09-13T15:08:47.555567+0000","last_clean":"2024-09-13T15:08:47.555567+0000","last_became_active":"2024-09-13T15:01:33.851887+0000","last_became_peered":"2024-09-13T15:01:33.851887+0000","last_unstale":"2024-09-13T15:08:47.555567+0000","last_undegraded":"2024-09-13T15:08:47.555567+0000","last_fullsized":"2024-09-13T15:08:47.555567+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:28.962060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.3","version":"445'13696","reported_seq":28881,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.828183+0000","last_change":"2024-09-13T15:08:52.582515+0000","last_active":"2024-09-13T15:08:52.828183+0000","last_peered":"2024-09-13T15:08:52.828183+0000","last_clean":"2024-09-13T15:08:52.828183+0000","last_became_active":"2024-09-13T15:08:38.479171+0000","last_became_peered":"2024-09-13T15:08:38.479171+0000","last_unstale":"2024-09-13T15:08:52.828183+0000","last_undegraded":"2024-09-13T15:08:52.828183+0000","last_fullsized":"2024-09-13T15:08:52.828183+0000","mapping_epoch":416,"log_start":"408'7726","ondisk_log_start":"408'7726","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:57.158293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54886400,"num_objects":837,"num_object_clones":0,"num_object_copies":2511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7064,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3756,"num_bytes_recovered":246153216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":163,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577997+0000","last_change":"2024-09-13T15:07:52.949239+0000","last_active":"2024-09-13T15:08:47.577997+0000","last_peered":"2024-09-13T15:08:47.577997+0000","last_clean":"2024-09-13T15:08:47.577997+0000","last_became_active":"2024-09-13T15:07:52.934781+0000","last_became_peered":"2024-09-13T15:07:52.934781+0000","last_unstale":"2024-09-13T15:08:47.577997+0000","last_undegraded":"2024-09-13T15:08:47.577997+0000","last_fullsized":"2024-09-13T15:08:47.577997+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:03:19.037785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":160,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578153+0000","last_change":"2024-09-13T15:08:09.184474+0000","last_active":"2024-09-13T15:08:47.578153+0000","last_peered":"2024-09-13T15:08:47.578153+0000","last_clean":"2024-09-13T15:08:47.578153+0000","last_became_active":"2024-09-13T15:08:09.184102+0000","last_became_peered":"2024-09-13T15:08:09.184102+0000","last_unstale":"2024-09-13T15:08:47.578153+0000","last_undegraded":"2024-09-13T15:08:47.578153+0000","last_fullsized":"2024-09-13T15:08:47.578153+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:52:33.422950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.0","version":"445'13261","reported_seq":13391,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:53.371813+0000","last_change":"2024-09-13T15:08:49.571927+0000","last_active":"2024-09-13T15:08:53.371813+0000","last_peered":"2024-09-13T15:08:53.371813+0000","last_clean":"2024-09-13T15:08:53.371813+0000","last_became_active":"2024-09-13T15:08:38.504355+0000","last_became_peered":"2024-09-13T15:08:38.504355+0000","last_unstale":"2024-09-13T15:08:53.371813+0000","last_undegraded":"2024-09-13T15:08:53.371813+0000","last_fullsized":"2024-09-13T15:08:53.371813+0000","mapping_epoch":395,"log_start":"404'5648","ondisk_log_start":"404'5648","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7613,"log_dups_size":0,"ondisk_log_size":7613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:54.547385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52002816,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6841,"num_write_kb":244320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":116,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555821+0000","last_change":"2024-09-13T15:01:33.853939+0000","last_active":"2024-09-13T15:08:47.555821+0000","last_peered":"2024-09-13T15:08:47.555821+0000","last_clean":"2024-09-13T15:08:47.555821+0000","last_became_active":"2024-09-13T15:01:33.853453+0000","last_became_peered":"2024-09-13T15:01:33.853453+0000","last_unstale":"2024-09-13T15:08:47.555821+0000","last_undegraded":"2024-09-13T15:08:47.555821+0000","last_fullsized":"2024-09-13T15:08:47.555821+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:26:23.516238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":439,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556141+0000","last_change":"2024-09-13T15:07:52.932503+0000","last_active":"2024-09-13T15:08:47.556141+0000","last_peered":"2024-09-13T15:08:47.556141+0000","last_clean":"2024-09-13T15:08:47.556141+0000","last_became_active":"2024-09-13T15:07:52.932202+0000","last_became_peered":"2024-09-13T15:07:52.932202+0000","last_unstale":"2024-09-13T15:08:47.556141+0000","last_undegraded":"2024-09-13T15:08:47.556141+0000","last_fullsized":"2024-09-13T15:08:47.556141+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:59.204152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.1","version":"445'13555","reported_seq":13699,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.807251+0000","last_change":"2024-09-13T15:08:50.148916+0000","last_active":"2024-09-13T15:08:52.807251+0000","last_peered":"2024-09-13T15:08:52.807251+0000","last_clean":"2024-09-13T15:08:52.807251+0000","last_became_active":"2024-09-13T15:08:38.515853+0000","last_became_peered":"2024-09-13T15:08:38.515853+0000","last_unstale":"2024-09-13T15:08:52.807251+0000","last_undegraded":"2024-09-13T15:08:52.807251+0000","last_fullsized":"2024-09-13T15:08:52.807251+0000","mapping_epoch":423,"log_start":"408'7612","ondisk_log_start":"408'7612","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5943,"log_dups_size":0,"ondisk_log_size":5943,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:55.135681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54165504,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6977,"num_write_kb":249728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":284,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584267+0000","last_change":"2024-09-13T15:01:33.855644+0000","last_active":"2024-09-13T15:08:52.584267+0000","last_peered":"2024-09-13T15:08:52.584267+0000","last_clean":"2024-09-13T15:08:52.584267+0000","last_became_active":"2024-09-13T15:01:33.855331+0000","last_became_peered":"2024-09-13T15:01:33.855331+0000","last_unstale":"2024-09-13T15:08:52.584267+0000","last_undegraded":"2024-09-13T15:08:52.584267+0000","last_fullsized":"2024-09-13T15:08:52.584267+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:56:44.847023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.581338+0000","last_change":"2024-09-13T15:07:52.937803+0000","last_active":"2024-09-13T15:08:51.581338+0000","last_peered":"2024-09-13T15:08:51.581338+0000","last_clean":"2024-09-13T15:08:51.581338+0000","last_became_active":"2024-09-13T15:07:52.935958+0000","last_became_peered":"2024-09-13T15:07:52.935958+0000","last_unstale":"2024-09-13T15:08:51.581338+0000","last_undegraded":"2024-09-13T15:08:51.581338+0000","last_fullsized":"2024-09-13T15:08:51.581338+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:36:15.989300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.6","version":"445'13679","reported_seq":21064,"reported_epoch":445,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-13T15:08:53.410175+0000","last_change":"2024-09-13T15:08:38.579842+0000","last_active":"2024-09-13T15:08:53.410175+0000","last_peered":"2024-09-13T15:08:53.410175+0000","last_clean":"2024-09-13T15:06:40.113598+0000","last_became_active":"2024-09-13T15:08:38.493992+0000","last_became_peered":"2024-09-13T15:08:38.493992+0000","last_unstale":"2024-09-13T15:08:53.410175+0000","last_undegraded":"2024-09-13T15:08:38.473768+0000","last_fullsized":"2024-09-13T15:08:38.468519+0000","mapping_epoch":407,"log_start":"404'6054","ondisk_log_start":"404'6054","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7625,"log_dups_size":0,"ondisk_log_size":7625,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51380224,"num_objects":784,"num_object_clones":0,"num_object_copies":2352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7065,"num_write_kb":251168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2580,"num_bytes_recovered":127631360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"2,3","objects":782}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":1070,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584259+0000","last_change":"2024-09-13T14:43:35.908842+0000","last_active":"2024-09-13T15:08:52.584259+0000","last_peered":"2024-09-13T15:08:52.584259+0000","last_clean":"2024-09-13T15:08:52.584259+0000","last_became_active":"2024-09-13T14:43:35.908500+0000","last_became_peered":"2024-09-13T14:43:35.908500+0000","last_unstale":"2024-09-13T15:08:52.584259+0000","last_undegraded":"2024-09-13T15:08:52.584259+0000","last_fullsized":"2024-09-13T15:08:52.584259+0000","mapping_epoch":156,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":157,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:13:28.916780+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":804,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582641+0000","last_change":"2024-09-13T15:06:40.613530+0000","last_active":"2024-09-13T15:08:52.582641+0000","last_peered":"2024-09-13T15:08:52.582641+0000","last_clean":"2024-09-13T15:08:52.582641+0000","last_became_active":"2024-09-13T15:06:40.608937+0000","last_became_peered":"2024-09-13T15:06:40.608937+0000","last_unstale":"2024-09-13T15:08:52.582641+0000","last_undegraded":"2024-09-13T15:08:52.582641+0000","last_fullsized":"2024-09-13T15:08:52.582641+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:10:49.728732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":133,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578183+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:08:47.578183+0000","last_peered":"2024-09-13T15:08:47.578183+0000","last_clean":"2024-09-13T15:08:47.578183+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:08:47.578183+0000","last_undegraded":"2024-09-13T15:08:47.578183+0000","last_fullsized":"2024-09-13T15:08:47.578183+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.7","version":"444'13423","reported_seq":13492,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.840194+0000","last_change":"2024-09-13T15:08:38.514372+0000","last_active":"2024-09-13T15:08:51.840194+0000","last_peered":"2024-09-13T15:08:51.840194+0000","last_clean":"2024-09-13T15:08:51.840194+0000","last_became_active":"2024-09-13T15:08:38.493255+0000","last_became_peered":"2024-09-13T15:08:38.493255+0000","last_unstale":"2024-09-13T15:08:51.840194+0000","last_undegraded":"2024-09-13T15:08:51.840194+0000","last_fullsized":"2024-09-13T15:08:51.840194+0000","mapping_epoch":395,"log_start":"404'6548","ondisk_log_start":"404'6548","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6875,"log_dups_size":0,"ondisk_log_size":6875,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53444608,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6931,"num_write_kb":247904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":568,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583136+0000","last_change":"2024-09-13T15:01:40.713718+0000","last_active":"2024-09-13T15:08:52.583136+0000","last_peered":"2024-09-13T15:08:52.583136+0000","last_clean":"2024-09-13T15:08:52.583136+0000","last_became_active":"2024-09-13T15:01:33.849650+0000","last_became_peered":"2024-09-13T15:01:33.849650+0000","last_unstale":"2024-09-13T15:08:52.583136+0000","last_undegraded":"2024-09-13T15:08:52.583136+0000","last_fullsized":"2024-09-13T15:08:52.583136+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:40.713573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:40.713573+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:47:23.136425+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":478,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556214+0000","last_change":"2024-09-13T15:06:40.605335+0000","last_active":"2024-09-13T15:08:47.556214+0000","last_peered":"2024-09-13T15:08:47.556214+0000","last_clean":"2024-09-13T15:08:47.556214+0000","last_became_active":"2024-09-13T15:06:40.604777+0000","last_became_peered":"2024-09-13T15:06:40.604777+0000","last_unstale":"2024-09-13T15:08:47.556214+0000","last_undegraded":"2024-09-13T15:08:47.556214+0000","last_fullsized":"2024-09-13T15:08:47.556214+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:00:26.565387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":151,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578237+0000","last_change":"2024-09-13T15:07:52.933247+0000","last_active":"2024-09-13T15:08:47.578237+0000","last_peered":"2024-09-13T15:08:47.578237+0000","last_clean":"2024-09-13T15:08:47.578237+0000","last_became_active":"2024-09-13T15:07:52.932736+0000","last_became_peered":"2024-09-13T15:07:52.932736+0000","last_unstale":"2024-09-13T15:08:47.578237+0000","last_undegraded":"2024-09-13T15:08:47.578237+0000","last_fullsized":"2024-09-13T15:08:47.578237+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:13:35.037964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.4","version":"444'13200","reported_seq":13269,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.978139+0000","last_change":"2024-09-13T15:08:38.524308+0000","last_active":"2024-09-13T15:08:51.978139+0000","last_peered":"2024-09-13T15:08:51.978139+0000","last_clean":"2024-09-13T15:08:51.978139+0000","last_became_active":"2024-09-13T15:08:38.502946+0000","last_became_peered":"2024-09-13T15:08:38.502946+0000","last_unstale":"2024-09-13T15:08:51.978139+0000","last_undegraded":"2024-09-13T15:08:51.978139+0000","last_fullsized":"2024-09-13T15:08:51.978139+0000","mapping_epoch":395,"log_start":"404'6161","ondisk_log_start":"404'6161","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7039,"log_dups_size":0,"ondisk_log_size":7039,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6809,"num_write_kb":244928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":133,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577716+0000","last_change":"2024-09-13T14:59:12.708929+0000","last_active":"2024-09-13T15:08:47.577716+0000","last_peered":"2024-09-13T15:08:47.577716+0000","last_clean":"2024-09-13T15:08:47.577716+0000","last_became_active":"2024-09-13T14:59:12.707047+0000","last_became_peered":"2024-09-13T14:59:12.707047+0000","last_unstale":"2024-09-13T15:08:47.577716+0000","last_undegraded":"2024-09-13T15:08:47.577716+0000","last_fullsized":"2024-09-13T15:08:47.577716+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:25.998843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":142,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578077+0000","last_change":"2024-09-13T15:06:40.605582+0000","last_active":"2024-09-13T15:08:47.578077+0000","last_peered":"2024-09-13T15:08:47.578077+0000","last_clean":"2024-09-13T15:08:47.578077+0000","last_became_active":"2024-09-13T15:06:40.605161+0000","last_became_peered":"2024-09-13T15:06:40.605161+0000","last_unstale":"2024-09-13T15:08:47.578077+0000","last_undegraded":"2024-09-13T15:08:47.578077+0000","last_fullsized":"2024-09-13T15:08:47.578077+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:11:32.977866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.5","version":"444'13533","reported_seq":13601,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.787997+0000","last_change":"2024-09-13T15:08:38.503975+0000","last_active":"2024-09-13T15:08:51.787997+0000","last_peered":"2024-09-13T15:08:51.787997+0000","last_clean":"2024-09-13T15:08:51.787997+0000","last_became_active":"2024-09-13T15:08:38.482590+0000","last_became_peered":"2024-09-13T15:08:38.482590+0000","last_unstale":"2024-09-13T15:08:51.787997+0000","last_undegraded":"2024-09-13T15:08:51.787997+0000","last_fullsized":"2024-09-13T15:08:51.787997+0000","mapping_epoch":395,"log_start":"405'6719","ondisk_log_start":"405'6719","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6814,"log_dups_size":0,"ondisk_log_size":6814,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53051392,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6975,"num_write_kb":249120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":568,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584213+0000","last_change":"2024-09-13T15:01:41.694383+0000","last_active":"2024-09-13T15:08:52.584213+0000","last_peered":"2024-09-13T15:08:52.584213+0000","last_clean":"2024-09-13T15:08:52.584213+0000","last_became_active":"2024-09-13T15:01:33.849983+0000","last_became_peered":"2024-09-13T15:01:33.849983+0000","last_unstale":"2024-09-13T15:08:52.584213+0000","last_undegraded":"2024-09-13T15:08:52.584213+0000","last_fullsized":"2024-09-13T15:08:52.584213+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:01:41.694296+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:01:41.694296+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:03:41.308064+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":743,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582774+0000","last_change":"2024-09-13T15:08:09.187080+0000","last_active":"2024-09-13T15:08:52.582774+0000","last_peered":"2024-09-13T15:08:52.582774+0000","last_clean":"2024-09-13T15:08:52.582774+0000","last_became_active":"2024-09-13T15:08:09.186862+0000","last_became_peered":"2024-09-13T15:08:09.186862+0000","last_unstale":"2024-09-13T15:08:52.582774+0000","last_undegraded":"2024-09-13T15:08:52.582774+0000","last_fullsized":"2024-09-13T15:08:52.582774+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:05:59.089697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.582478+0000","last_change":"2024-09-13T15:07:52.946957+0000","last_active":"2024-09-13T15:08:51.582478+0000","last_peered":"2024-09-13T15:08:51.582478+0000","last_clean":"2024-09-13T15:08:51.582478+0000","last_became_active":"2024-09-13T15:07:52.946389+0000","last_became_peered":"2024-09-13T15:07:52.946389+0000","last_unstale":"2024-09-13T15:08:51.582478+0000","last_undegraded":"2024-09-13T15:08:51.582478+0000","last_fullsized":"2024-09-13T15:08:51.582478+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.963362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.f","version":"445'13864","reported_seq":16015,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.854057+0000","last_change":"2024-09-13T15:07:52.974299+0000","last_active":"2024-09-13T15:08:52.854057+0000","last_peered":"2024-09-13T15:08:52.854057+0000","last_clean":"2024-09-13T15:08:52.854057+0000","last_became_active":"2024-09-13T15:07:52.956988+0000","last_became_peered":"2024-09-13T15:07:52.956988+0000","last_unstale":"2024-09-13T15:08:52.854057+0000","last_undegraded":"2024-09-13T15:08:52.854057+0000","last_fullsized":"2024-09-13T15:08:52.854057+0000","mapping_epoch":422,"log_start":"417'8966","ondisk_log_start":"417'8966","created":395,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":4898,"log_dups_size":0,"ondisk_log_size":4898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:00:37.929524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13864,"num_write_kb":492672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1010,"num_bytes_recovered":66125824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":688,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555752+0000","last_change":"2024-09-13T15:08:09.189415+0000","last_active":"2024-09-13T15:08:47.555752+0000","last_peered":"2024-09-13T15:08:47.555752+0000","last_clean":"2024-09-13T15:08:47.555752+0000","last_became_active":"2024-09-13T15:08:09.188726+0000","last_became_peered":"2024-09-13T15:08:09.188726+0000","last_unstale":"2024-09-13T15:08:47.555752+0000","last_undegraded":"2024-09-13T15:08:47.555752+0000","last_fullsized":"2024-09-13T15:08:47.555752+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:00:00.397088+0000","scrub_duration":0.0014177090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":206,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584556+0000","last_change":"2024-09-13T15:01:33.851374+0000","last_active":"2024-09-13T15:08:52.584556+0000","last_peered":"2024-09-13T15:08:52.584556+0000","last_clean":"2024-09-13T15:08:52.584556+0000","last_became_active":"2024-09-13T15:01:33.850998+0000","last_became_peered":"2024-09-13T15:01:33.850998+0000","last_unstale":"2024-09-13T15:08:52.584556+0000","last_undegraded":"2024-09-13T15:08:52.584556+0000","last_fullsized":"2024-09-13T15:08:52.584556+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:55.071461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":707,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583014+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:08:52.583014+0000","last_peered":"2024-09-13T15:08:52.583014+0000","last_clean":"2024-09-13T15:08:52.583014+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:08:52.583014+0000","last_undegraded":"2024-09-13T15:08:52.583014+0000","last_fullsized":"2024-09-13T15:08:52.583014+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":430,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555981+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:08:47.555981+0000","last_peered":"2024-09-13T15:08:47.555981+0000","last_clean":"2024-09-13T15:08:47.555981+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:08:47.555981+0000","last_undegraded":"2024-09-13T15:08:47.555981+0000","last_fullsized":"2024-09-13T15:08:47.555981+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"14.c","version":"444'13879","reported_seq":13938,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.849711+0000","last_change":"2024-09-13T15:05:11.104165+0000","last_active":"2024-09-13T15:08:51.849711+0000","last_peered":"2024-09-13T15:08:51.849711+0000","last_clean":"2024-09-13T15:08:51.849711+0000","last_became_active":"2024-09-13T15:05:11.103885+0000","last_became_peered":"2024-09-13T15:05:11.103885+0000","last_unstale":"2024-09-13T15:08:51.849711+0000","last_undegraded":"2024-09-13T15:08:51.849711+0000","last_fullsized":"2024-09-13T15:08:51.849711+0000","mapping_epoch":395,"log_start":"409'7944","ondisk_log_start":"409'7944","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:58:32.640246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107937792,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13879,"num_write_kb":496832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":797,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.582317+0000","last_change":"2024-09-13T15:07:55.932598+0000","last_active":"2024-09-13T15:08:51.582317+0000","last_peered":"2024-09-13T15:08:51.582317+0000","last_clean":"2024-09-13T15:08:51.582317+0000","last_became_active":"2024-09-13T15:07:55.932277+0000","last_became_peered":"2024-09-13T15:07:55.932277+0000","last_unstale":"2024-09-13T15:08:51.582317+0000","last_undegraded":"2024-09-13T15:08:51.582317+0000","last_fullsized":"2024-09-13T15:08:51.582317+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:56.613460+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":660,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582195+0000","last_change":"2024-09-13T15:06:40.607795+0000","last_active":"2024-09-13T15:08:52.582195+0000","last_peered":"2024-09-13T15:08:52.582195+0000","last_clean":"2024-09-13T15:08:52.582195+0000","last_became_active":"2024-09-13T15:06:40.607431+0000","last_became_peered":"2024-09-13T15:06:40.607431+0000","last_unstale":"2024-09-13T15:08:52.582195+0000","last_undegraded":"2024-09-13T15:08:52.582195+0000","last_fullsized":"2024-09-13T15:08:52.582195+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:19:47.696723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577543+0000","last_change":"2024-09-13T15:03:44.115491+0000","last_active":"2024-09-13T15:08:47.577543+0000","last_peered":"2024-09-13T15:08:47.577543+0000","last_clean":"2024-09-13T15:08:47.577543+0000","last_became_active":"2024-09-13T15:03:44.114978+0000","last_became_peered":"2024-09-13T15:03:44.114978+0000","last_unstale":"2024-09-13T15:08:47.577543+0000","last_undegraded":"2024-09-13T15:08:47.577543+0000","last_fullsized":"2024-09-13T15:08:47.577543+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:28:04.996891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.582103+0000","last_change":"2024-09-13T15:07:52.944954+0000","last_active":"2024-09-13T15:08:51.582103+0000","last_peered":"2024-09-13T15:08:51.582103+0000","last_clean":"2024-09-13T15:08:51.582103+0000","last_became_active":"2024-09-13T15:07:52.944654+0000","last_became_peered":"2024-09-13T15:07:52.944654+0000","last_unstale":"2024-09-13T15:08:51.582103+0000","last_undegraded":"2024-09-13T15:08:51.582103+0000","last_fullsized":"2024-09-13T15:08:51.582103+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:50.580294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.d","version":"444'13991","reported_seq":14051,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.971982+0000","last_change":"2024-09-13T15:05:11.537213+0000","last_active":"2024-09-13T15:08:51.971982+0000","last_peered":"2024-09-13T15:08:51.971982+0000","last_clean":"2024-09-13T15:08:51.971982+0000","last_became_active":"2024-09-13T15:05:11.536909+0000","last_became_peered":"2024-09-13T15:05:11.536909+0000","last_unstale":"2024-09-13T15:08:51.971982+0000","last_undegraded":"2024-09-13T15:08:51.971982+0000","last_fullsized":"2024-09-13T15:08:51.971982+0000","mapping_epoch":395,"log_start":"409'7920","ondisk_log_start":"409'7920","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:13.844350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512984,"num_objects":1611,"num_object_clones":0,"num_object_copies":4833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1611,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13991,"num_write_kb":499201,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":693,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577743+0000","last_change":"2024-09-13T15:03:46.571609+0000","last_active":"2024-09-13T15:08:47.577743+0000","last_peered":"2024-09-13T15:08:47.577743+0000","last_clean":"2024-09-13T15:08:47.577743+0000","last_became_active":"2024-09-13T15:03:46.571111+0000","last_became_peered":"2024-09-13T15:03:46.571111+0000","last_unstale":"2024-09-13T15:08:47.577743+0000","last_undegraded":"2024-09-13T15:08:47.577743+0000","last_fullsized":"2024-09-13T15:08:47.577743+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:26.923160+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":791,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583772+0000","last_change":"2024-09-13T15:07:52.935042+0000","last_active":"2024-09-13T15:08:52.583772+0000","last_peered":"2024-09-13T15:08:52.583772+0000","last_clean":"2024-09-13T15:08:52.583772+0000","last_became_active":"2024-09-13T15:07:52.934697+0000","last_became_peered":"2024-09-13T15:07:52.934697+0000","last_unstale":"2024-09-13T15:08:52.583772+0000","last_undegraded":"2024-09-13T15:08:52.583772+0000","last_fullsized":"2024-09-13T15:08:52.583772+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:09:23.505991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.581393+0000","last_change":"2024-09-13T15:07:52.938622+0000","last_active":"2024-09-13T15:08:51.581393+0000","last_peered":"2024-09-13T15:08:51.581393+0000","last_clean":"2024-09-13T15:08:51.581393+0000","last_became_active":"2024-09-13T15:07:52.938323+0000","last_became_peered":"2024-09-13T15:07:52.938323+0000","last_unstale":"2024-09-13T15:08:51.581393+0000","last_undegraded":"2024-09-13T15:08:51.581393+0000","last_fullsized":"2024-09-13T15:08:51.581393+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:53:29.800644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":89,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582936+0000","last_change":"2024-09-13T15:06:40.606903+0000","last_active":"2024-09-13T15:08:52.582936+0000","last_peered":"2024-09-13T15:08:52.582936+0000","last_clean":"2024-09-13T15:08:52.582936+0000","last_became_active":"2024-09-13T15:06:40.606596+0000","last_became_peered":"2024-09-13T15:06:40.606596+0000","last_unstale":"2024-09-13T15:08:52.582936+0000","last_undegraded":"2024-09-13T15:08:52.582936+0000","last_fullsized":"2024-09-13T15:08:52.582936+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:17:44.157857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.e","version":"444'13419","reported_seq":13478,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.828943+0000","last_change":"2024-09-13T15:05:11.104779+0000","last_active":"2024-09-13T15:08:51.828943+0000","last_peered":"2024-09-13T15:08:51.828943+0000","last_clean":"2024-09-13T15:08:51.828943+0000","last_became_active":"2024-09-13T15:05:11.104444+0000","last_became_peered":"2024-09-13T15:05:11.104444+0000","last_unstale":"2024-09-13T15:08:51.828943+0000","last_undegraded":"2024-09-13T15:08:51.828943+0000","last_fullsized":"2024-09-13T15:08:51.828943+0000","mapping_epoch":395,"log_start":"408'7444","ondisk_log_start":"408'7444","created":395,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5975,"log_dups_size":0,"ondisk_log_size":5975,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:36:23.452855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13419,"num_write_kb":480896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":531,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583512+0000","last_change":"2024-09-13T15:03:47.227943+0000","last_active":"2024-09-13T15:08:52.583512+0000","last_peered":"2024-09-13T15:08:52.583512+0000","last_clean":"2024-09-13T15:08:52.583512+0000","last_became_active":"2024-09-13T15:03:46.585701+0000","last_became_peered":"2024-09-13T15:03:46.585701+0000","last_unstale":"2024-09-13T15:08:52.583512+0000","last_undegraded":"2024-09-13T15:08:52.583512+0000","last_fullsized":"2024-09-13T15:08:52.583512+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:03:47.226199+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:03:47.226199+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:59:12.460412+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":58,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.584881+0000","last_change":"2024-09-13T15:08:37.806317+0000","last_active":"2024-09-13T15:08:51.584881+0000","last_peered":"2024-09-13T15:08:51.584881+0000","last_clean":"2024-09-13T15:08:51.584881+0000","last_became_active":"2024-09-13T15:07:52.939036+0000","last_became_peered":"2024-09-13T15:07:52.939036+0000","last_unstale":"2024-09-13T15:08:51.584881+0000","last_undegraded":"2024-09-13T15:08:51.584881+0000","last_fullsized":"2024-09-13T15:08:51.584881+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:28:24.428749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":180,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578126+0000","last_change":"2024-09-13T15:08:09.186888+0000","last_active":"2024-09-13T15:08:47.578126+0000","last_peered":"2024-09-13T15:08:47.578126+0000","last_clean":"2024-09-13T15:08:47.578126+0000","last_became_active":"2024-09-13T15:08:09.186416+0000","last_became_peered":"2024-09-13T15:08:09.186416+0000","last_unstale":"2024-09-13T15:08:47.578126+0000","last_undegraded":"2024-09-13T15:08:47.578126+0000","last_fullsized":"2024-09-13T15:08:47.578126+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:54:56.667822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":177,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.585697+0000","last_change":"2024-09-13T15:06:40.605296+0000","last_active":"2024-09-13T15:08:52.585697+0000","last_peered":"2024-09-13T15:08:52.585697+0000","last_clean":"2024-09-13T15:08:52.585697+0000","last_became_active":"2024-09-13T15:06:40.604969+0000","last_became_peered":"2024-09-13T15:06:40.604969+0000","last_unstale":"2024-09-13T15:08:52.585697+0000","last_undegraded":"2024-09-13T15:08:52.585697+0000","last_fullsized":"2024-09-13T15:08:52.585697+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:01:00.071208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"14.8","version":"445'13357","reported_seq":15794,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:53.392462+0000","last_change":"2024-09-13T15:08:47.651448+0000","last_active":"2024-09-13T15:08:53.392462+0000","last_peered":"2024-09-13T15:08:53.392462+0000","last_clean":"2024-09-13T15:08:53.392462+0000","last_became_active":"2024-09-13T15:08:47.555325+0000","last_became_peered":"2024-09-13T15:08:47.555325+0000","last_unstale":"2024-09-13T15:08:53.392462+0000","last_undegraded":"2024-09-13T15:08:53.392462+0000","last_fullsized":"2024-09-13T15:08:53.392462+0000","mapping_epoch":442,"log_start":"404'5700","ondisk_log_start":"404'5700","created":395,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7657,"log_dups_size":0,"ondisk_log_size":7657,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53215232,"num_objects":812,"num_object_clones":0,"num_object_copies":2436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6893,"num_write_kb":246560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1132,"num_bytes_recovered":74219520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":697,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556071+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:08:47.556071+0000","last_peered":"2024-09-13T15:08:47.556071+0000","last_clean":"2024-09-13T15:08:47.556071+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:08:47.556071+0000","last_undegraded":"2024-09-13T15:08:47.556071+0000","last_fullsized":"2024-09-13T15:08:47.556071+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583907+0000","last_change":"2024-09-13T15:01:33.854558+0000","last_active":"2024-09-13T15:08:52.583907+0000","last_peered":"2024-09-13T15:08:52.583907+0000","last_clean":"2024-09-13T15:08:52.583907+0000","last_became_active":"2024-09-13T15:01:33.854038+0000","last_became_peered":"2024-09-13T15:01:33.854038+0000","last_unstale":"2024-09-13T15:08:52.583907+0000","last_undegraded":"2024-09-13T15:08:52.583907+0000","last_fullsized":"2024-09-13T15:08:52.583907+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:18:22.961881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.581750+0000","last_change":"2024-09-13T15:07:52.933730+0000","last_active":"2024-09-13T15:08:51.581750+0000","last_peered":"2024-09-13T15:08:51.581750+0000","last_clean":"2024-09-13T15:08:51.581750+0000","last_became_active":"2024-09-13T15:07:52.931809+0000","last_became_peered":"2024-09-13T15:07:52.931809+0000","last_unstale":"2024-09-13T15:08:51.581750+0000","last_undegraded":"2024-09-13T15:08:51.581750+0000","last_fullsized":"2024-09-13T15:08:51.581750+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:27:23.226452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":456,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577599+0000","last_change":"2024-09-13T15:06:40.601744+0000","last_active":"2024-09-13T15:08:47.577599+0000","last_peered":"2024-09-13T15:08:47.577599+0000","last_clean":"2024-09-13T15:08:47.577599+0000","last_became_active":"2024-09-13T15:06:40.601439+0000","last_became_peered":"2024-09-13T15:06:40.601439+0000","last_unstale":"2024-09-13T15:08:47.577599+0000","last_undegraded":"2024-09-13T15:08:47.577599+0000","last_fullsized":"2024-09-13T15:08:47.577599+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:12:47.746856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.9","version":"445'13440","reported_seq":13571,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.868358+0000","last_change":"2024-09-13T15:08:51.172299+0000","last_active":"2024-09-13T15:08:52.868358+0000","last_peered":"2024-09-13T15:08:52.868358+0000","last_clean":"2024-09-13T15:08:52.868358+0000","last_became_active":"2024-09-13T15:08:38.481466+0000","last_became_peered":"2024-09-13T15:08:38.481466+0000","last_unstale":"2024-09-13T15:08:52.868358+0000","last_undegraded":"2024-09-13T15:08:52.868358+0000","last_fullsized":"2024-09-13T15:08:52.868358+0000","mapping_epoch":395,"log_start":"408'7498","ondisk_log_start":"408'7498","created":395,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:56.142739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50036736,"num_objects":763,"num_object_clones":0,"num_object_copies":2289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6938,"num_write_kb":246464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":763,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556767+0000","last_change":"2024-09-13T15:03:46.574886+0000","last_active":"2024-09-13T15:08:47.556767+0000","last_peered":"2024-09-13T15:08:47.556767+0000","last_clean":"2024-09-13T15:08:47.556767+0000","last_became_active":"2024-09-13T15:03:46.574563+0000","last_became_peered":"2024-09-13T15:03:46.574563+0000","last_unstale":"2024-09-13T15:08:47.556767+0000","last_undegraded":"2024-09-13T15:08:47.556767+0000","last_fullsized":"2024-09-13T15:08:47.556767+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:16:29.856598+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.585511+0000","last_change":"2024-09-13T15:08:09.186120+0000","last_active":"2024-09-13T15:08:52.585511+0000","last_peered":"2024-09-13T15:08:52.585511+0000","last_clean":"2024-09-13T15:08:52.585511+0000","last_became_active":"2024-09-13T15:08:09.185851+0000","last_became_peered":"2024-09-13T15:08:09.185851+0000","last_unstale":"2024-09-13T15:08:52.585511+0000","last_undegraded":"2024-09-13T15:08:52.585511+0000","last_fullsized":"2024-09-13T15:08:52.585511+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:04:22.561596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":264,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578459+0000","last_change":"2024-09-13T15:08:09.187714+0000","last_active":"2024-09-13T15:08:47.578459+0000","last_peered":"2024-09-13T15:08:47.578459+0000","last_clean":"2024-09-13T15:08:47.578459+0000","last_became_active":"2024-09-13T15:08:09.187404+0000","last_became_peered":"2024-09-13T15:08:09.187404+0000","last_unstale":"2024-09-13T15:08:47.578459+0000","last_undegraded":"2024-09-13T15:08:47.578459+0000","last_fullsized":"2024-09-13T15:08:47.578459+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:04.738653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.582414+0000","last_change":"2024-09-13T15:07:52.933604+0000","last_active":"2024-09-13T15:08:51.582414+0000","last_peered":"2024-09-13T15:08:51.582414+0000","last_clean":"2024-09-13T15:08:51.582414+0000","last_became_active":"2024-09-13T15:07:52.933192+0000","last_became_peered":"2024-09-13T15:07:52.933192+0000","last_unstale":"2024-09-13T15:08:51.582414+0000","last_undegraded":"2024-09-13T15:08:51.582414+0000","last_fullsized":"2024-09-13T15:08:51.582414+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:53:05.303014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.a","version":"444'14118","reported_seq":14235,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.984332+0000","last_change":"2024-09-13T15:07:53.951600+0000","last_active":"2024-09-13T15:08:51.984332+0000","last_peered":"2024-09-13T15:08:51.984332+0000","last_clean":"2024-09-13T15:08:51.984332+0000","last_became_active":"2024-09-13T15:07:53.925119+0000","last_became_peered":"2024-09-13T15:07:53.925119+0000","last_unstale":"2024-09-13T15:08:51.984332+0000","last_undegraded":"2024-09-13T15:08:51.984332+0000","last_fullsized":"2024-09-13T15:08:51.984332+0000","mapping_epoch":423,"log_start":"404'5951","ondisk_log_start":"404'5951","created":395,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":8167,"log_dups_size":0,"ondisk_log_size":8167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:30.100649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108134400,"num_objects":1650,"num_object_clones":0,"num_object_copies":4950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14118,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":626,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578391+0000","last_change":"2024-09-13T14:59:15.702205+0000","last_active":"2024-09-13T15:08:47.578391+0000","last_peered":"2024-09-13T15:08:47.578391+0000","last_clean":"2024-09-13T15:08:47.578391+0000","last_became_active":"2024-09-13T14:59:15.701862+0000","last_became_peered":"2024-09-13T14:59:15.701862+0000","last_unstale":"2024-09-13T15:08:47.578391+0000","last_undegraded":"2024-09-13T15:08:47.578391+0000","last_fullsized":"2024-09-13T15:08:47.578391+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:46:56.790225+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":752,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583414+0000","last_change":"2024-09-13T15:03:44.131836+0000","last_active":"2024-09-13T15:08:52.583414+0000","last_peered":"2024-09-13T15:08:52.583414+0000","last_clean":"2024-09-13T15:08:52.583414+0000","last_became_active":"2024-09-13T15:03:44.120633+0000","last_became_peered":"2024-09-13T15:03:44.120633+0000","last_unstale":"2024-09-13T15:08:52.583414+0000","last_undegraded":"2024-09-13T15:08:52.583414+0000","last_fullsized":"2024-09-13T15:08:52.583414+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:01:52.065604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":531,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.587883+0000","last_change":"2024-09-13T15:06:40.605165+0000","last_active":"2024-09-13T15:08:51.587883+0000","last_peered":"2024-09-13T15:08:51.587883+0000","last_clean":"2024-09-13T15:08:51.587883+0000","last_became_active":"2024-09-13T15:06:40.604503+0000","last_became_peered":"2024-09-13T15:06:40.604503+0000","last_unstale":"2024-09-13T15:08:51.587883+0000","last_undegraded":"2024-09-13T15:08:51.587883+0000","last_fullsized":"2024-09-13T15:08:51.587883+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:45:57.829820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":766,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582524+0000","last_change":"2024-09-13T15:07:52.933408+0000","last_active":"2024-09-13T15:08:52.582524+0000","last_peered":"2024-09-13T15:08:52.582524+0000","last_clean":"2024-09-13T15:08:52.582524+0000","last_became_active":"2024-09-13T15:07:52.933015+0000","last_became_peered":"2024-09-13T15:07:52.933015+0000","last_unstale":"2024-09-13T15:08:52.582524+0000","last_undegraded":"2024-09-13T15:08:52.582524+0000","last_fullsized":"2024-09-13T15:08:52.582524+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:59:51.373304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.b","version":"445'13853","reported_seq":13984,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:53.404643+0000","last_change":"2024-09-13T15:07:53.953162+0000","last_active":"2024-09-13T15:08:53.404643+0000","last_peered":"2024-09-13T15:08:53.404643+0000","last_clean":"2024-09-13T15:08:53.404643+0000","last_became_active":"2024-09-13T15:07:53.926761+0000","last_became_peered":"2024-09-13T15:07:53.926761+0000","last_unstale":"2024-09-13T15:08:53.404643+0000","last_undegraded":"2024-09-13T15:08:53.404643+0000","last_fullsized":"2024-09-13T15:08:53.404643+0000","mapping_epoch":423,"log_start":"405'7017","ondisk_log_start":"405'7017","created":395,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:02:10.237702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104529920,"num_objects":1595,"num_object_clones":0,"num_object_copies":4785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13853,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":981,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583983+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:08:52.583983+0000","last_peered":"2024-09-13T15:08:52.583983+0000","last_clean":"2024-09-13T15:08:52.583983+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:08:52.583983+0000","last_undegraded":"2024-09-13T15:08:52.583983+0000","last_fullsized":"2024-09-13T15:08:52.583983+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":348,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578166+0000","last_change":"2024-09-13T15:08:09.190731+0000","last_active":"2024-09-13T15:08:47.578166+0000","last_peered":"2024-09-13T15:08:47.578166+0000","last_clean":"2024-09-13T15:08:47.578166+0000","last_became_active":"2024-09-13T15:08:09.187657+0000","last_became_peered":"2024-09-13T15:08:09.187657+0000","last_unstale":"2024-09-13T15:08:47.578166+0000","last_undegraded":"2024-09-13T15:08:47.578166+0000","last_fullsized":"2024-09-13T15:08:47.578166+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:16:44.137504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":434,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556015+0000","last_change":"2024-09-13T14:40:02.370184+0000","last_active":"2024-09-13T15:08:47.556015+0000","last_peered":"2024-09-13T15:08:47.556015+0000","last_clean":"2024-09-13T15:08:47.556015+0000","last_became_active":"2024-09-13T14:40:02.369870+0000","last_became_peered":"2024-09-13T14:40:02.369870+0000","last_unstale":"2024-09-13T15:08:47.556015+0000","last_undegraded":"2024-09-13T15:08:47.556015+0000","last_fullsized":"2024-09-13T15:08:47.556015+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:41:36.826418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":133,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577559+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:08:47.577559+0000","last_peered":"2024-09-13T15:08:47.577559+0000","last_clean":"2024-09-13T15:08:47.577559+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:08:47.577559+0000","last_undegraded":"2024-09-13T15:08:47.577559+0000","last_fullsized":"2024-09-13T15:08:47.577559+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.19","version":"445'13446","reported_seq":13574,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.857354+0000","last_change":"2024-09-13T15:08:49.156714+0000","last_active":"2024-09-13T15:08:52.857354+0000","last_peered":"2024-09-13T15:08:52.857354+0000","last_clean":"2024-09-13T15:08:52.857354+0000","last_became_active":"2024-09-13T15:08:38.555210+0000","last_became_peered":"2024-09-13T15:08:38.555210+0000","last_unstale":"2024-09-13T15:08:52.857354+0000","last_undegraded":"2024-09-13T15:08:52.857354+0000","last_fullsized":"2024-09-13T15:08:52.857354+0000","mapping_epoch":436,"log_start":"408'7498","ondisk_log_start":"408'7498","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:54.112925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49643520,"num_objects":758,"num_object_clones":0,"num_object_copies":2274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6945,"num_write_kb":246464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578207+0000","last_change":"2024-09-13T15:08:09.188203+0000","last_active":"2024-09-13T15:08:47.578207+0000","last_peered":"2024-09-13T15:08:47.578207+0000","last_clean":"2024-09-13T15:08:47.578207+0000","last_became_active":"2024-09-13T15:08:09.187271+0000","last_became_peered":"2024-09-13T15:08:09.187271+0000","last_unstale":"2024-09-13T15:08:47.578207+0000","last_undegraded":"2024-09-13T15:08:47.578207+0000","last_fullsized":"2024-09-13T15:08:47.578207+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:45:31.670696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":143,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578088+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:08:47.578088+0000","last_peered":"2024-09-13T15:08:47.578088+0000","last_clean":"2024-09-13T15:08:47.578088+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:08:47.578088+0000","last_undegraded":"2024-09-13T15:08:47.578088+0000","last_fullsized":"2024-09-13T15:08:47.578088+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.18","version":"445'13391","reported_seq":13561,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-13T15:08:53.350085+0000","last_change":"2024-09-13T15:08:40.534437+0000","last_active":"2024-09-13T15:08:53.350085+0000","last_peered":"2024-09-13T15:08:53.350085+0000","last_clean":"2024-09-13T15:06:40.084226+0000","last_became_active":"2024-09-13T15:08:40.500766+0000","last_became_peered":"2024-09-13T15:08:40.500766+0000","last_unstale":"2024-09-13T15:08:53.350085+0000","last_undegraded":"2024-09-13T15:08:53.350085+0000","last_fullsized":"2024-09-13T15:08:53.350085+0000","mapping_epoch":438,"log_start":"404'5700","ondisk_log_start":"404'5700","created":436,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7691,"log_dups_size":0,"ondisk_log_size":7691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":775,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6927,"num_write_kb":246560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":622592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,1],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":778}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":126,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.556222+0000","last_change":"2024-09-13T15:06:40.601737+0000","last_active":"2024-09-13T15:08:47.556222+0000","last_peered":"2024-09-13T15:08:47.556222+0000","last_clean":"2024-09-13T15:08:47.556222+0000","last_became_active":"2024-09-13T15:06:40.601141+0000","last_became_peered":"2024-09-13T15:06:40.601141+0000","last_unstale":"2024-09-13T15:08:47.556222+0000","last_undegraded":"2024-09-13T15:08:47.556222+0000","last_fullsized":"2024-09-13T15:08:47.556222+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:35:56.285800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":694,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584797+0000","last_change":"2024-09-13T14:59:12.717095+0000","last_active":"2024-09-13T15:08:52.584797+0000","last_peered":"2024-09-13T15:08:52.584797+0000","last_clean":"2024-09-13T15:08:52.584797+0000","last_became_active":"2024-09-13T14:59:12.710176+0000","last_became_peered":"2024-09-13T14:59:12.710176+0000","last_unstale":"2024-09-13T15:08:52.584797+0000","last_undegraded":"2024-09-13T15:08:52.584797+0000","last_fullsized":"2024-09-13T15:08:52.584797+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:08.617733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":88,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582812+0000","last_change":"2024-09-13T15:06:40.606724+0000","last_active":"2024-09-13T15:08:52.582812+0000","last_peered":"2024-09-13T15:08:52.582812+0000","last_clean":"2024-09-13T15:08:52.582812+0000","last_became_active":"2024-09-13T15:06:40.606228+0000","last_became_peered":"2024-09-13T15:06:40.606228+0000","last_unstale":"2024-09-13T15:08:52.582812+0000","last_undegraded":"2024-09-13T15:08:52.582812+0000","last_fullsized":"2024-09-13T15:08:52.582812+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:32:21.085510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555891+0000","last_change":"2024-09-13T15:01:33.851738+0000","last_active":"2024-09-13T15:08:47.555891+0000","last_peered":"2024-09-13T15:08:47.555891+0000","last_clean":"2024-09-13T15:08:47.555891+0000","last_became_active":"2024-09-13T15:01:33.851321+0000","last_became_peered":"2024-09-13T15:01:33.851321+0000","last_unstale":"2024-09-13T15:08:47.555891+0000","last_undegraded":"2024-09-13T15:08:47.555891+0000","last_fullsized":"2024-09-13T15:08:47.555891+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:19:08.607464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,7],"acting":[6,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":347,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583257+0000","last_change":"2024-09-13T15:03:44.113646+0000","last_active":"2024-09-13T15:08:52.583257+0000","last_peered":"2024-09-13T15:08:52.583257+0000","last_clean":"2024-09-13T15:08:52.583257+0000","last_became_active":"2024-09-13T15:03:44.113342+0000","last_became_peered":"2024-09-13T15:03:44.113342+0000","last_unstale":"2024-09-13T15:08:52.583257+0000","last_undegraded":"2024-09-13T15:08:52.583257+0000","last_fullsized":"2024-09-13T15:08:52.583257+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:06:29.444600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":740,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582434+0000","last_change":"2024-09-13T15:08:09.184381+0000","last_active":"2024-09-13T15:08:52.582434+0000","last_peered":"2024-09-13T15:08:52.582434+0000","last_clean":"2024-09-13T15:08:52.582434+0000","last_became_active":"2024-09-13T15:08:09.184077+0000","last_became_peered":"2024-09-13T15:08:09.184077+0000","last_unstale":"2024-09-13T15:08:52.582434+0000","last_undegraded":"2024-09-13T15:08:52.582434+0000","last_fullsized":"2024-09-13T15:08:52.582434+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:13:26.703635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.15","version":"444'13508","reported_seq":13576,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.802589+0000","last_change":"2024-09-13T15:08:38.558819+0000","last_active":"2024-09-13T15:08:51.802589+0000","last_peered":"2024-09-13T15:08:51.802589+0000","last_clean":"2024-09-13T15:08:51.802589+0000","last_became_active":"2024-09-13T15:08:38.533724+0000","last_became_peered":"2024-09-13T15:08:38.533724+0000","last_unstale":"2024-09-13T15:08:51.802589+0000","last_undegraded":"2024-09-13T15:08:51.802589+0000","last_fullsized":"2024-09-13T15:08:51.802589+0000","mapping_epoch":436,"log_start":"405'6719","ondisk_log_start":"405'6719","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6789,"log_dups_size":0,"ondisk_log_size":6789,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54689792,"num_objects":835,"num_object_clones":0,"num_object_copies":2505,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":835,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6951,"num_write_kb":249120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":440,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.555920+0000","last_change":"2024-09-13T15:07:52.936799+0000","last_active":"2024-09-13T15:08:47.555920+0000","last_peered":"2024-09-13T15:08:47.555920+0000","last_clean":"2024-09-13T15:08:47.555920+0000","last_became_active":"2024-09-13T15:07:52.936436+0000","last_became_peered":"2024-09-13T15:07:52.936436+0000","last_unstale":"2024-09-13T15:08:47.555920+0000","last_undegraded":"2024-09-13T15:08:47.555920+0000","last_fullsized":"2024-09-13T15:08:47.555920+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:52:09.480999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":787,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.582672+0000","last_change":"2024-09-13T15:06:40.608166+0000","last_active":"2024-09-13T15:08:52.582672+0000","last_peered":"2024-09-13T15:08:52.582672+0000","last_clean":"2024-09-13T15:08:52.582672+0000","last_became_active":"2024-09-13T15:06:40.607830+0000","last_became_peered":"2024-09-13T15:06:40.607830+0000","last_unstale":"2024-09-13T15:08:52.582672+0000","last_undegraded":"2024-09-13T15:08:52.582672+0000","last_fullsized":"2024-09-13T15:08:52.582672+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:32:48.177335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.14","version":"444'13173","reported_seq":13242,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.958283+0000","last_change":"2024-09-13T15:08:38.559980+0000","last_active":"2024-09-13T15:08:51.958283+0000","last_peered":"2024-09-13T15:08:51.958283+0000","last_clean":"2024-09-13T15:08:51.958283+0000","last_became_active":"2024-09-13T15:08:38.537347+0000","last_became_peered":"2024-09-13T15:08:38.537347+0000","last_unstale":"2024-09-13T15:08:51.958283+0000","last_undegraded":"2024-09-13T15:08:51.958283+0000","last_fullsized":"2024-09-13T15:08:51.958283+0000","mapping_epoch":436,"log_start":"404'6161","ondisk_log_start":"404'6161","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7012,"log_dups_size":0,"ondisk_log_size":7012,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57147392,"num_objects":872,"num_object_clones":0,"num_object_copies":2616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6782,"num_write_kb":244928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":503,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583042+0000","last_change":"2024-09-13T15:07:52.941232+0000","last_active":"2024-09-13T15:08:52.583042+0000","last_peered":"2024-09-13T15:08:52.583042+0000","last_clean":"2024-09-13T15:08:52.583042+0000","last_became_active":"2024-09-13T15:07:52.939661+0000","last_became_peered":"2024-09-13T15:07:52.939661+0000","last_unstale":"2024-09-13T15:08:52.583042+0000","last_undegraded":"2024-09-13T15:08:52.583042+0000","last_fullsized":"2024-09-13T15:08:52.583042+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:32:42.857066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.582150+0000","last_change":"2024-09-13T15:07:52.936887+0000","last_active":"2024-09-13T15:08:51.582150+0000","last_peered":"2024-09-13T15:08:51.582150+0000","last_clean":"2024-09-13T15:08:51.582150+0000","last_became_active":"2024-09-13T15:07:52.936398+0000","last_became_peered":"2024-09-13T15:07:52.936398+0000","last_unstale":"2024-09-13T15:08:51.582150+0000","last_undegraded":"2024-09-13T15:08:51.582150+0000","last_fullsized":"2024-09-13T15:08:51.582150+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:05.892375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"14.17","version":"444'13372","reported_seq":13441,"reported_epoch":444,"state":"active+clean","last_fresh":"2024-09-13T15:08:51.804793+0000","last_change":"2024-09-13T15:08:38.535489+0000","last_active":"2024-09-13T15:08:51.804793+0000","last_peered":"2024-09-13T15:08:51.804793+0000","last_clean":"2024-09-13T15:08:51.804793+0000","last_became_active":"2024-09-13T15:08:38.514550+0000","last_became_peered":"2024-09-13T15:08:38.514550+0000","last_unstale":"2024-09-13T15:08:51.804793+0000","last_undegraded":"2024-09-13T15:08:51.804793+0000","last_fullsized":"2024-09-13T15:08:51.804793+0000","mapping_epoch":436,"log_start":"404'6548","ondisk_log_start":"404'6548","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56786944,"num_objects":867,"num_object_clones":0,"num_object_copies":2601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6881,"num_write_kb":247904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":746,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.583681+0000","last_change":"2024-09-13T15:07:52.935989+0000","last_active":"2024-09-13T15:08:52.583681+0000","last_peered":"2024-09-13T15:08:52.583681+0000","last_clean":"2024-09-13T15:08:52.583681+0000","last_became_active":"2024-09-13T15:07:52.935661+0000","last_became_peered":"2024-09-13T15:07:52.935661+0000","last_unstale":"2024-09-13T15:08:52.583681+0000","last_undegraded":"2024-09-13T15:08:52.583681+0000","last_fullsized":"2024-09-13T15:08:52.583681+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:49.027141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":150,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578914+0000","last_change":"2024-09-13T15:07:52.935604+0000","last_active":"2024-09-13T15:08:47.578914+0000","last_peered":"2024-09-13T15:08:47.578914+0000","last_clean":"2024-09-13T15:08:47.578914+0000","last_became_active":"2024-09-13T15:07:52.932711+0000","last_became_peered":"2024-09-13T15:07:52.932711+0000","last_unstale":"2024-09-13T15:08:47.578914+0000","last_undegraded":"2024-09-13T15:08:47.578914+0000","last_fullsized":"2024-09-13T15:08:47.578914+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:06.384328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.16","version":"445'13685","reported_seq":18003,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-13T15:08:53.349164+0000","last_change":"2024-09-13T15:08:40.568825+0000","last_active":"2024-09-13T15:08:53.349164+0000","last_peered":"2024-09-13T15:08:53.349164+0000","last_clean":"2024-09-13T15:06:40.113598+0000","last_became_active":"2024-09-13T15:08:40.503764+0000","last_became_peered":"2024-09-13T15:08:40.503764+0000","last_unstale":"2024-09-13T15:08:53.349164+0000","last_undegraded":"2024-09-13T15:08:40.500927+0000","last_fullsized":"2024-09-13T15:08:40.483327+0000","mapping_epoch":438,"log_start":"404'6054","ondisk_log_start":"404'6054","created":436,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7631,"log_dups_size":0,"ondisk_log_size":7631,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50987008,"num_objects":778,"num_object_clones":0,"num_object_copies":2334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2544,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7071,"num_write_kb":251168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1049,"num_bytes_recovered":68714496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3],"avail_no_missing":["2","1","3"],"object_location_counts":[{"shards":"1,2,3","objects":465},{"shards":"2,3","objects":-144}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":758,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.585607+0000","last_change":"2024-09-13T15:07:52.931529+0000","last_active":"2024-09-13T15:08:52.585607+0000","last_peered":"2024-09-13T15:08:52.585607+0000","last_clean":"2024-09-13T15:08:52.585607+0000","last_became_active":"2024-09-13T15:07:52.931055+0000","last_became_peered":"2024-09-13T15:07:52.931055+0000","last_unstale":"2024-09-13T15:08:52.585607+0000","last_undegraded":"2024-09-13T15:08:52.585607+0000","last_fullsized":"2024-09-13T15:08:52.585607+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:07:02.788954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":128,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578038+0000","last_change":"2024-09-13T15:06:40.605841+0000","last_active":"2024-09-13T15:08:47.578038+0000","last_peered":"2024-09-13T15:08:47.578038+0000","last_clean":"2024-09-13T15:08:47.578038+0000","last_became_active":"2024-09-13T15:06:40.604704+0000","last_became_peered":"2024-09-13T15:06:40.604704+0000","last_unstale":"2024-09-13T15:08:47.578038+0000","last_undegraded":"2024-09-13T15:08:47.578038+0000","last_fullsized":"2024-09-13T15:08:47.578038+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:51:58.305980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.11","version":"445'13559","reported_seq":13693,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.873434+0000","last_change":"2024-09-13T15:08:48.180659+0000","last_active":"2024-09-13T15:08:52.873434+0000","last_peered":"2024-09-13T15:08:52.873434+0000","last_clean":"2024-09-13T15:08:52.873434+0000","last_became_active":"2024-09-13T15:08:38.555165+0000","last_became_peered":"2024-09-13T15:08:38.555165+0000","last_unstale":"2024-09-13T15:08:52.873434+0000","last_undegraded":"2024-09-13T15:08:52.873434+0000","last_fullsized":"2024-09-13T15:08:52.873434+0000","mapping_epoch":436,"log_start":"408'7612","ondisk_log_start":"408'7612","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:53.119742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53903360,"num_objects":823,"num_object_clones":0,"num_object_copies":2469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6982,"num_write_kb":249728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":150,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577950+0000","last_change":"2024-09-13T15:06:40.603223+0000","last_active":"2024-09-13T15:08:47.577950+0000","last_peered":"2024-09-13T15:08:47.577950+0000","last_clean":"2024-09-13T15:08:47.577950+0000","last_became_active":"2024-09-13T15:06:40.602930+0000","last_became_peered":"2024-09-13T15:06:40.602930+0000","last_unstale":"2024-09-13T15:08:47.577950+0000","last_undegraded":"2024-09-13T15:08:47.577950+0000","last_fullsized":"2024-09-13T15:08:47.577950+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:18:16.757274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":440,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578001+0000","last_change":"2024-09-13T15:07:52.943245+0000","last_active":"2024-09-13T15:08:47.578001+0000","last_peered":"2024-09-13T15:08:47.578001+0000","last_clean":"2024-09-13T15:08:47.578001+0000","last_became_active":"2024-09-13T15:07:52.932624+0000","last_became_peered":"2024-09-13T15:07:52.932624+0000","last_unstale":"2024-09-13T15:08:47.578001+0000","last_undegraded":"2024-09-13T15:08:47.578001+0000","last_fullsized":"2024-09-13T15:08:47.578001+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:20:50.061001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.10","version":"445'13288","reported_seq":13418,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:53.359737+0000","last_change":"2024-09-13T15:08:50.561796+0000","last_active":"2024-09-13T15:08:53.359737+0000","last_peered":"2024-09-13T15:08:53.359737+0000","last_clean":"2024-09-13T15:08:53.359737+0000","last_became_active":"2024-09-13T15:08:38.534406+0000","last_became_peered":"2024-09-13T15:08:38.534406+0000","last_unstale":"2024-09-13T15:08:53.359737+0000","last_undegraded":"2024-09-13T15:08:53.359737+0000","last_fullsized":"2024-09-13T15:08:53.359737+0000","mapping_epoch":436,"log_start":"404'5648","ondisk_log_start":"404'5648","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7640,"log_dups_size":0,"ondisk_log_size":7640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-13T15:08:55.535596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50233344,"num_objects":767,"num_object_clones":0,"num_object_copies":2301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6869,"num_write_kb":244320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":152,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577603+0000","last_change":"2024-09-13T15:03:44.116627+0000","last_active":"2024-09-13T15:08:47.577603+0000","last_peered":"2024-09-13T15:08:47.577603+0000","last_clean":"2024-09-13T15:08:47.577603+0000","last_became_active":"2024-09-13T15:03:44.116197+0000","last_became_peered":"2024-09-13T15:03:44.116197+0000","last_unstale":"2024-09-13T15:08:47.577603+0000","last_undegraded":"2024-09-13T15:08:47.577603+0000","last_fullsized":"2024-09-13T15:08:47.577603+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:28:30.519779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":399,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.578315+0000","last_change":"2024-09-13T15:01:33.856627+0000","last_active":"2024-09-13T15:08:47.578315+0000","last_peered":"2024-09-13T15:08:47.578315+0000","last_clean":"2024-09-13T15:08:47.578315+0000","last_became_active":"2024-09-13T15:01:33.856245+0000","last_became_peered":"2024-09-13T15:01:33.856245+0000","last_unstale":"2024-09-13T15:08:47.578315+0000","last_undegraded":"2024-09-13T15:08:47.578315+0000","last_fullsized":"2024-09-13T15:08:47.578315+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":355,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:58:02.228050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"14.13","version":"445'13688","reported_seq":28870,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.844909+0000","last_change":"2024-09-13T15:08:47.215548+0000","last_active":"2024-09-13T15:08:52.844909+0000","last_peered":"2024-09-13T15:08:52.844909+0000","last_clean":"2024-09-13T15:08:52.844909+0000","last_became_active":"2024-09-13T15:08:38.517805+0000","last_became_peered":"2024-09-13T15:08:38.517805+0000","last_unstale":"2024-09-13T15:08:52.844909+0000","last_undegraded":"2024-09-13T15:08:52.844909+0000","last_fullsized":"2024-09-13T15:08:52.844909+0000","mapping_epoch":436,"log_start":"408'7726","ondisk_log_start":"408'7726","created":436,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55410688,"num_objects":846,"num_object_clones":0,"num_object_copies":2538,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":846,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7057,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3756,"num_bytes_recovered":246153216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":782,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584610+0000","last_change":"2024-09-13T15:03:44.120794+0000","last_active":"2024-09-13T15:08:52.584610+0000","last_peered":"2024-09-13T15:08:52.584610+0000","last_clean":"2024-09-13T15:08:52.584610+0000","last_became_active":"2024-09-13T15:03:44.120431+0000","last_became_peered":"2024-09-13T15:03:44.120431+0000","last_unstale":"2024-09-13T15:08:52.584610+0000","last_undegraded":"2024-09-13T15:08:52.584610+0000","last_fullsized":"2024-09-13T15:08:52.584610+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:39:27.396753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":742,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.584080+0000","last_change":"2024-09-13T14:40:37.186884+0000","last_active":"2024-09-13T15:08:52.584080+0000","last_peered":"2024-09-13T15:08:52.584080+0000","last_clean":"2024-09-13T15:08:52.584080+0000","last_became_active":"2024-09-13T14:40:37.186440+0000","last_became_peered":"2024-09-13T14:40:37.186440+0000","last_unstale":"2024-09-13T15:08:52.584080+0000","last_undegraded":"2024-09-13T15:08:52.584080+0000","last_fullsized":"2024-09-13T15:08:52.584080+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:44:22.548462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"14.12","version":"443'13507","reported_seq":27599,"reported_epoch":444,"state":"peering","last_fresh":"2024-09-13T15:08:51.586337+0000","last_change":"2024-09-13T15:08:51.586526+0000","last_active":"2024-09-13T15:08:51.586337+0000","last_peered":"2024-09-13T15:08:51.535517+0000","last_clean":"2024-09-13T15:08:51.535517+0000","last_became_active":"2024-09-13T15:08:40.492893+0000","last_became_peered":"2024-09-13T15:08:40.492893+0000","last_unstale":"2024-09-13T15:08:51.586337+0000","last_undegraded":"2024-09-13T15:08:51.586337+0000","last_fullsized":"2024-09-13T15:08:51.586337+0000","mapping_epoch":444,"log_start":"404'6505","ondisk_log_start":"404'6505","created":436,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T15:05:10.082604+0000","last_clean_scrub_stamp":"2024-09-13T15:05:10.082604+0000","objects_scrubbed":0,"log_size":7002,"log_dups_size":0,"ondisk_log_size":7002,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6948,"num_write_kb":249216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3966,"num_bytes_recovered":259883008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":150,"reported_epoch":443,"state":"active+clean","last_fresh":"2024-09-13T15:08:47.577530+0000","last_change":"2024-09-13T15:07:52.933733+0000","last_active":"2024-09-13T15:08:47.577530+0000","last_peered":"2024-09-13T15:08:47.577530+0000","last_clean":"2024-09-13T15:08:47.577530+0000","last_became_active":"2024-09-13T15:07:52.933425+0000","last_became_peered":"2024-09-13T15:07:52.933425+0000","last_unstale":"2024-09-13T15:08:47.577530+0000","last_undegraded":"2024-09-13T15:08:47.577530+0000","last_fullsized":"2024-09-13T15:08:47.577530+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:59.564486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":177,"reported_epoch":445,"state":"active+clean","last_fresh":"2024-09-13T15:08:52.585717+0000","last_change":"2024-09-13T15:03:44.120052+0000","last_active":"2024-09-13T15:08:52.585717+0000","last_peered":"2024-09-13T15:08:52.585717+0000","last_clean":"2024-09-13T15:08:52.585717+0000","last_became_active":"2024-09-13T15:03:44.109611+0000","last_became_peered":"2024-09-13T15:03:44.109611+0000","last_unstale":"2024-09-13T15:08:52.585717+0000","last_undegraded":"2024-09-13T15:08:52.585717+0000","last_fullsized":"2024-09-13T15:08:52.585717+0000","mapping_epoch":372,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:04:55.196794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":67,"num_bytes_recovered":5808,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":14,"num_pg":26,"stat_sum":{"num_bytes":1698234392,"num_objects":25914,"num_object_clones":0,"num_object_copies":77742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2546,"num_objects_misplaced":775,"num_objects_unfound":0,"num_objects_dirty":25914,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":221996,"num_write_kb":7933057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21521,"num_bytes_recovered":1368391680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5736509440,"data_stored":5736497224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":174826,"ondisk_log_size":174826,"up":78,"acting":76,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234586,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777212,"kb_used_data":362292,"kb_used_omap":1,"kb_used_meta":1414910,"kb_avail":91967940,"statfs":{"total":95995035648,"available":94175170560,"internally_reserved":0,"allocated":370987008,"data_stored":368612436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1448868299},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496223,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2239096,"kb_used_data":821804,"kb_used_omap":1,"kb_used_meta":1417278,"kb_avail":91506056,"statfs":{"total":95995035648,"available":93702201344,"internally_reserved":0,"allocated":841527296,"data_stored":839153155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1451293127},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757862,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2318840,"kb_used_data":679088,"kb_used_omap":1,"kb_used_meta":1639742,"kb_avail":91426312,"statfs":{"total":95995035648,"available":93620543488,"internally_reserved":0,"allocated":695386112,"data_stored":693011684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1679096262},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019497,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2085480,"kb_used_data":754208,"kb_used_omap":1,"kb_used_meta":1331262,"kb_avail":91659672,"statfs":{"total":95995035648,"available":93859504128,"internally_reserved":0,"allocated":772308992,"data_stored":769938263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1363212742},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650709,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2200804,"kb_used_data":1055960,"kb_used_omap":4,"kb_used_meta":1144827,"kb_avail":91544348,"statfs":{"total":95995035648,"available":93741412352,"internally_reserved":0,"allocated":1081303040,"data_stored":1078944574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":1172303407},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879644,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848232,"kb_used_data":740064,"kb_used_omap":6,"kb_used_meta":1108153,"kb_avail":91896920,"statfs":{"total":95995035648,"available":94102446080,"internally_reserved":0,"allocated":757825536,"data_stored":755459494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":1134749241},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435661,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2663048,"kb_used_data":1361280,"kb_used_omap":5,"kb_used_meta":1301754,"kb_avail":91082104,"statfs":{"total":95995035648,"available":93268074496,"internally_reserved":0,"allocated":1393950720,"data_stored":1391607318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1332996449},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":1494648619105,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1364590592,"data_stored":1364590592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":721616896,"data_stored":721616896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1059913728,"data_stored":1059913728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":750784512,"data_stored":750780440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":653463552,"data_stored":653459480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":828903424,"data_stored":828899352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":14,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":357236736,"data_stored":357236736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:08:59.878 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-13T15:08:59.878 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_9 pg_num 36 2024-09-13T15:08:59.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:08:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:08:59.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:08:59.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[108570]: 14.8 scrub starts 2024-09-13T15:08:59.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[108570]: pgmap v1582: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.8 MiB/s, 197 objects/s recovering 2024-09-13T15:08:59.936 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:08:59.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[113926]: 14.8 scrub starts 2024-09-13T15:08:59.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[113926]: pgmap v1582: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.44k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.8 MiB/s, 197 objects/s recovering 2024-09-13T15:08:59.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:08:59.937 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:08:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:08:59] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.51.0" 2024-09-13T15:09:00.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:00.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:00.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:00.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:00.178 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:00.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:00.280 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:00.317 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:00.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:00.419 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:00.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:00.546 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.583 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:00.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:00.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.714 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:00.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:00.816 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:00.852 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:00.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:00.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:00.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:00 smithi042 ceph-mon[108570]: 14.11 scrub starts 2024-09-13T15:09:00.900 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:00 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2546/60186 objects degraded (4.230%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:00.901 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:00 smithi042 ceph-mon[113926]: 14.11 scrub starts 2024-09-13T15:09:00.901 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:00 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2546/60186 objects degraded (4.230%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:00.909 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:00.954 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:00.995 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:09:00.995 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:00 smithi159 ceph-mon[102519]: 14.11 scrub starts 2024-09-13T15:09:00.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:00 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2546/60186 objects degraded (4.230%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:01.012 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:01.029 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:09:01.029 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:01.056 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:09:01.056 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:01.089 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:09:01.089 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:01.113 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:01.122 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:09:01.122 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:01.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:01.152 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:09:01.152 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:01.190 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:09:01.190 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:01.224 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:09:01.224 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:01.252 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:01.285 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:01.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:01.329 INFO:teuthology.orchestra.run.smithi159.stdout:ERROR: (22) Invalid argument 2024-09-13T15:09:01.329 INFO:teuthology.orchestra.run.smithi159.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:09:01.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:01.422 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:01.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:01.524 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:01.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.560 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:01.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:01.661 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:01.691 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:01.793 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:01.829 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:09:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[113926]: pgmap v1583: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.1 MiB/s, 171 objects/s recovering 2024-09-13T15:09:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1537290981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:01.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[113926]: 14.19 scrub starts 2024-09-13T15:09:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:09:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[108570]: pgmap v1583: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.1 MiB/s, 171 objects/s recovering 2024-09-13T15:09:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1537290981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:01.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:01 smithi042 ceph-mon[108570]: 14.19 scrub starts 2024-09-13T15:09:01.932 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:01.965 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:01.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:01.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:01.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:01 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:09:01.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:01 smithi159 ceph-mon[102519]: pgmap v1583: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/60186 objects degraded (4.230%); 556/60186 objects misplaced (0.924%); 5.1 MiB/s, 171 objects/s recovering 2024-09-13T15:09:01.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1537290981' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:01.996 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:01 smithi159 ceph-mon[102519]: 14.19 scrub starts 2024-09-13T15:09:02.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:02.110 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:02.212 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:02.244 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:02.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:02.346 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:02.360 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:09:02.378 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:02.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:02.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:02.506 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:02.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.529 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:02.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:02.629 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:02.660 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:02 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:09:02.661 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:02 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:09:02.661 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:02 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:09:02.661 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:02 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:09:02.673 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:02.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:02.775 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:02.812 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:02.914 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:02.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:02.957 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:02.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:02 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:09:02.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:02 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:09:03.058 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:03.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:03.194 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.245 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.346 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.384 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.486 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:03.527 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:03.628 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:03.664 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:03 smithi159 ceph-mon[102519]: pgmap v1584: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2546/50637 objects degraded (5.028%); 556/50637 objects misplaced (1.098%); 0 B/s, 81 objects/s recovering 2024-09-13T15:09:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:03 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:09:03.766 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:03.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.805 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:03.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:03 smithi042 ceph-mon[108570]: pgmap v1584: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2546/50637 objects degraded (5.028%); 556/50637 objects misplaced (1.098%); 0 B/s, 81 objects/s recovering 2024-09-13T15:09:03.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:03 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:09:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:03 smithi042 ceph-mon[113926]: pgmap v1584: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 1.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 2546/50637 objects degraded (5.028%); 556/50637 objects misplaced (1.098%); 0 B/s, 81 objects/s recovering 2024-09-13T15:09:03.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:03 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:09:03.907 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:03.942 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:03.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:03.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:04.045 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:04.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:09:04.082 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:04.184 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.216 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:04.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:04.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:04.354 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:04.458 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:04.510 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:04.612 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.638 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:04.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:04.739 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:04.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:04.878 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:04.909 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:04.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:04.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:04.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:04 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:09:05.011 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.046 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:05.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:05.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:04 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:09:05.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:04 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:09:05.147 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:05.185 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:05.287 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:05.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.324 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:05.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:05.425 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:05.463 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:05.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:05.540 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:05.565 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:05.626 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:05.667 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:09:05.729 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:05.761 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:05.863 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:05.908 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:05.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: 14.8 scrub starts 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: pgmap v1585: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 101 objects/s recovering 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/744231611' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:09:05.924 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 2546/50637 objects degraded (5.028%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: 14.8 scrub starts 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: pgmap v1585: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 101 objects/s recovering 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/744231611' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:09:05.925 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:05 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 2546/50637 objects degraded (5.028%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: 14.8 scrub starts 2024-09-13T15:09:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: pgmap v1585: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 101 objects/s recovering 2024-09-13T15:09:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/744231611' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:05.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:06.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:09:06.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:05 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 2546/50637 objects degraded (5.028%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T15:09:06.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:06.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.049 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:06.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:06.150 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.184 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:06.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:06.285 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.325 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:06.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:06.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.461 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:06.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:06.561 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.600 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:06.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:06.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:06.731 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:06.833 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:06.871 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:06.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:06.973 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: osdmap e446: 8 total, 8 up, 7 in 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: 14.11 scrub starts 2024-09-13T15:09:06.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:06 smithi159 ceph-mon[102519]: 14.11 scrub ok 2024-09-13T15:09:07.010 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:07.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: osdmap e446: 8 total, 8 up, 7 in 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: 14.11 scrub starts 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[108570]: 14.11 scrub ok 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:07.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: osdmap e446: 8 total, 8 up, 7 in 2024-09-13T15:09:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: 14.11 scrub starts 2024-09-13T15:09:07.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:06 smithi042 ceph-mon[113926]: 14.11 scrub ok 2024-09-13T15:09:07.111 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.144 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:07.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:07.201 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:07.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:07.298 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:07.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.444 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:07.546 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:07.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.582 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:07.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:07.683 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.726 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:07.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[113926]: 14.0 scrub ok 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[113926]: pgmap v1587: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 72 objects/s recovering 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[113926]: 14.19 scrub starts 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[113926]: 14.19 scrub ok 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:09:07.798 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[108570]: 14.0 scrub ok 2024-09-13T15:09:07.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[108570]: pgmap v1587: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 72 objects/s recovering 2024-09-13T15:09:07.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[108570]: 14.19 scrub starts 2024-09-13T15:09:07.799 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:07 smithi042 ceph-mon[108570]: 14.19 scrub ok 2024-09-13T15:09:07.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:07.861 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:07.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:07.963 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:07.989 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:07.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:07 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:09:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:07 smithi159 ceph-mon[102519]: 14.0 scrub ok 2024-09-13T15:09:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:07 smithi159 ceph-mon[102519]: pgmap v1587: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 856 MiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 2546/41124 objects degraded (6.191%); 366/41124 objects misplaced (0.890%); 0 B/s, 72 objects/s recovering 2024-09-13T15:09:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:07 smithi159 ceph-mon[102519]: 14.19 scrub starts 2024-09-13T15:09:07.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:07 smithi159 ceph-mon[102519]: 14.19 scrub ok 2024-09-13T15:09:08.091 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:08.113 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:08.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:08.215 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:08.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.236 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:08.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:08.338 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:08.359 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:08.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:08.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:08.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:08.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:08.607 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:08.676 INFO:teuthology.orchestra.run.smithi042.stderr:set pool 14 pg_num to 36 2024-09-13T15:09:08.709 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:08.741 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:08.843 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:08.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: 14.10 scrub starts 2024-09-13T15:09:08.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: 14.10 scrub ok 2024-09-13T15:09:08.848 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: osdmap e447: 8 total, 8 up, 7 in 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2403980242' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[113926]: 14.1 scrub ok 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: 14.10 scrub starts 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: 14.10 scrub ok 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: osdmap e447: 8 total, 8 up, 7 in 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2403980242' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.850 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:08 smithi042 ceph-mon[108570]: 14.1 scrub ok 2024-09-13T15:09:08.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:08.879 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:08.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:08.980 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: 14.10 scrub starts 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: 14.10 scrub ok 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: osdmap e447: 8 total, 8 up, 7 in 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2403980242' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"} : dispatch 2024-09-13T15:09:08.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:08 smithi159 ceph-mon[102519]: 14.1 scrub ok 2024-09-13T15:09:09.020 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:09.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:09.122 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.156 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:09.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:09.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:09.288 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:09.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:09.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:09.425 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:09.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:09.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:09.564 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:09.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:09.666 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:09.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:09.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.834 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:09.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:09.935 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:09.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:09.970 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:09.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2024-09-13T15:09:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:09 smithi159 ceph-mon[102519]: osdmap e448: 8 total, 8 up, 7 in 2024-09-13T15:09:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:09 smithi159 ceph-mon[102519]: pgmap v1590: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 144 objects/s recovering 2024-09-13T15:09:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:09 smithi159 ceph-mon[102519]: 14.13 scrub starts 2024-09-13T15:09:09.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:09 smithi159 ceph-mon[102519]: 14.13 scrub ok 2024-09-13T15:09:10.071 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2024-09-13T15:09:10.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[113926]: osdmap e448: 8 total, 8 up, 7 in 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[113926]: pgmap v1590: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 144 objects/s recovering 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[113926]: 14.13 scrub starts 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[113926]: 14.13 scrub ok 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:09] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num", "val": "36"}]': finished 2024-09-13T15:09:10.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[108570]: osdmap e448: 8 total, 8 up, 7 in 2024-09-13T15:09:10.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[108570]: pgmap v1590: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 144 objects/s recovering 2024-09-13T15:09:10.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[108570]: 14.13 scrub starts 2024-09-13T15:09:10.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:09 smithi042 ceph-mon[108570]: 14.13 scrub ok 2024-09-13T15:09:10.111 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:10.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:10.119 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:09:10.119 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:09:10.212 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:10.353 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:10.389 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:10.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:10.492 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:10.513 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:10.615 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:10.652 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:10.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:10.754 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:10.780 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:10.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:10.919 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:10.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:10.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.5 scrub starts 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.5 scrub ok 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.8 scrub starts 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.8 scrub ok 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:09:10.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: 14.3 scrub ok 2024-09-13T15:09:11.000 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:10 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 922/22074 objects degraded (4.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:11.021 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.058 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:11.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.5 scrub starts 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.5 scrub ok 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.8 scrub starts 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.8 scrub ok 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: 14.3 scrub ok 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 922/22074 objects degraded (4.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.5 scrub starts 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.5 scrub ok 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.8 scrub starts 2024-09-13T15:09:11.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.8 scrub ok 2024-09-13T15:09:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:09:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: 14.3 scrub ok 2024-09-13T15:09:11.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:10 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 922/22074 objects degraded (4.177%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:11.160 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:11.202 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:11.304 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:11.337 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:11.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:11.472 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:11.574 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:11.609 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:11.712 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:11.742 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:11.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:11.844 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:11.886 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:11.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:11.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:11 smithi159 ceph-mon[102519]: 14.6 scrub starts 2024-09-13T15:09:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:11 smithi159 ceph-mon[102519]: 14.6 scrub ok 2024-09-13T15:09:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:11 smithi159 ceph-mon[102519]: pgmap v1591: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 80 objects/s recovering 2024-09-13T15:09:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:11 smithi159 ceph-mon[102519]: 14.9 scrub starts 2024-09-13T15:09:11.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:11 smithi159 ceph-mon[102519]: 14.9 scrub ok 2024-09-13T15:09:12.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:12.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[108570]: 14.6 scrub starts 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[108570]: 14.6 scrub ok 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[108570]: pgmap v1591: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 80 objects/s recovering 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[108570]: 14.9 scrub starts 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[108570]: 14.9 scrub ok 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[113926]: 14.6 scrub starts 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[113926]: 14.6 scrub ok 2024-09-13T15:09:12.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[113926]: pgmap v1591: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 459 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/22074 objects degraded (4.177%); 183/22074 objects misplaced (0.829%); 0 B/s, 80 objects/s recovering 2024-09-13T15:09:12.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[113926]: 14.9 scrub starts 2024-09-13T15:09:12.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:11 smithi042 ceph-mon[113926]: 14.9 scrub ok 2024-09-13T15:09:12.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:12.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:12.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:12.278 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:12.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:12.380 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:12.416 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:12.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:12.518 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.543 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:12.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:12.644 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:12.679 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:12.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:12.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:12.814 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:12.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:12.916 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:12.951 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:12.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:12.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:12 smithi159 ceph-mon[102519]: 14.7 scrub starts 2024-09-13T15:09:12.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:12 smithi159 ceph-mon[102519]: 14.7 scrub ok 2024-09-13T15:09:13.053 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.089 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:13.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:12 smithi042 ceph-mon[108570]: 14.7 scrub starts 2024-09-13T15:09:13.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:12 smithi042 ceph-mon[108570]: 14.7 scrub ok 2024-09-13T15:09:13.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:12 smithi042 ceph-mon[113926]: 14.7 scrub starts 2024-09-13T15:09:13.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:12 smithi042 ceph-mon[113926]: 14.7 scrub ok 2024-09-13T15:09:13.190 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:13.230 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:13.332 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:13.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:13.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:13.469 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:13.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:13.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:13.640 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:13.744 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:13.775 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:13.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:13.877 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:13.918 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:13.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:13.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:13 smithi159 ceph-mon[102519]: pgmap v1592: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 302 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/14571 objects degraded (6.328%); 183/14571 objects misplaced (1.256%); 0 B/s, 68 objects/s recovering 2024-09-13T15:09:14.019 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.054 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:14.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:13 smithi042 ceph-mon[108570]: pgmap v1592: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 302 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/14571 objects degraded (6.328%); 183/14571 objects misplaced (1.256%); 0 B/s, 68 objects/s recovering 2024-09-13T15:09:14.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:13 smithi042 ceph-mon[113926]: pgmap v1592: 109 pgs: 1 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 106 active+clean; 302 MiB data, 11 GiB used, 615 GiB / 626 GiB avail; 922/14571 objects degraded (6.328%); 183/14571 objects misplaced (1.256%); 0 B/s, 68 objects/s recovering 2024-09-13T15:09:14.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:14.191 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:14.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:14.217 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Removed 123954 objects 2024-09-13T15:09:14.217 INFO:tasks.radosbench.radosbench.0.smithi042.stdout:Clean up completed and total clean up time :138.73 2024-09-13T15:09:14.293 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:14.314 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:14.416 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:14.437 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:14.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:14.539 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:14.560 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:14.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:14.662 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:14.685 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:14.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:14.786 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:14.810 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:14.911 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:14.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:14.980 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:14.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:15.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:15.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T15:09:15.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T15:09:15.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:15.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:15.230 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:15.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"} : dispatch 2024-09-13T15:09:15.332 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:15.354 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:15.456 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:15.478 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:15.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:15.580 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:15.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.604 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:15.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:15.705 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:15.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:15.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.851 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:15.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:15.952 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:15.977 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:15.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:16.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[108570]: pgmap v1593: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:16.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[108570]: osdmap e449: 8 total, 8 up, 7 in 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 922/6345 objects degraded (14.531%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[113926]: pgmap v1593: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[113926]: osdmap e449: 8 total, 8 up, 7 in 2024-09-13T15:09:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:15 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 922/6345 objects degraded (14.531%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:16.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:16.202 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:16.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.226 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:16.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:15 smithi159 ceph-mon[102519]: pgmap v1593: 109 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:15 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pg_num_actual", "val": "36"}]': finished 2024-09-13T15:09:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:15 smithi159 ceph-mon[102519]: osdmap e449: 8 total, 8 up, 7 in 2024-09-13T15:09:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:15 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 922/6345 objects degraded (14.531%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:16.327 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:16.348 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:16.450 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:16.477 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:16.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:16.579 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:16.602 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:16.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:16.704 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:16.726 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:16.828 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:16.850 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:16.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:16.952 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:16.973 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:16.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:16.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:17.076 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:17.097 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:17.144 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: osdmap e450: 8 total, 8 up, 7 in 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.1 scrub starts 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.1 scrub ok 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.a scrub starts 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.a scrub ok 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.0 scrub starts 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[108570]: 14.0 scrub ok 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: osdmap e450: 8 total, 8 up, 7 in 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.1 scrub starts 2024-09-13T15:09:17.145 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.1 scrub ok 2024-09-13T15:09:17.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.a scrub starts 2024-09-13T15:09:17.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.a scrub ok 2024-09-13T15:09:17.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.0 scrub starts 2024-09-13T15:09:17.146 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:16 smithi042 ceph-mon[113926]: 14.0 scrub ok 2024-09-13T15:09:17.199 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.225 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:17.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: osdmap e450: 8 total, 8 up, 7 in 2024-09-13T15:09:17.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.1 scrub starts 2024-09-13T15:09:17.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.1 scrub ok 2024-09-13T15:09:17.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.a scrub starts 2024-09-13T15:09:17.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.a scrub ok 2024-09-13T15:09:17.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.0 scrub starts 2024-09-13T15:09:17.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:16 smithi159 ceph-mon[102519]: 14.0 scrub ok 2024-09-13T15:09:17.326 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.350 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:17.451 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:17.474 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:17.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:17.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:17.701 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:17.723 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:17.825 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:17.847 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:17.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:17.972 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:17.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:18.074 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.095 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:18.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:18.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:18.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:18.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: pgmap v1596: 119 pgs: 10 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.f scrub starts 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.f scrub ok 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.1a scrub starts 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.1a scrub ok 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.20 scrub starts 2024-09-13T15:09:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:17 smithi159 ceph-mon[102519]: 14.20 scrub ok 2024-09-13T15:09:18.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:18.346 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: pgmap v1596: 119 pgs: 10 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.f scrub starts 2024-09-13T15:09:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.f scrub ok 2024-09-13T15:09:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.1a scrub starts 2024-09-13T15:09:18.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.1a scrub ok 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.20 scrub starts 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[108570]: 14.20 scrub ok 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: pgmap v1596: 119 pgs: 10 unknown, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 107 active+clean; 131 MiB data, 10 GiB used, 616 GiB / 626 GiB avail; 922/6345 objects degraded (14.531%); 8/6345 objects misplaced (0.126%) 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.f scrub starts 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.f scrub ok 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.1a scrub starts 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.1a scrub ok 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.20 scrub starts 2024-09-13T15:09:18.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:17 smithi042 ceph-mon[113926]: 14.20 scrub ok 2024-09-13T15:09:18.448 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:18.471 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:18.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:18.573 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:18.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.595 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:18.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:18.696 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:18.718 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:18.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:18.819 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:18.840 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:18.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:18.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:18.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:18.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:18.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:19.040 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_9 2024-09-13T15:09:19.040 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_9 unique_pool_9 --yes-i-really-really-mean-it 2024-09-13T15:09:19.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:19.106 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[113926]: 14.3 scrub starts 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[113926]: 14.3 scrub ok 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[113926]: 14.b scrub starts 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[113926]: 14.b scrub ok 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[108570]: 14.3 scrub starts 2024-09-13T15:09:19.187 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[108570]: 14.3 scrub ok 2024-09-13T15:09:19.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[108570]: 14.b scrub starts 2024-09-13T15:09:19.188 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:18 smithi042 ceph-mon[108570]: 14.b scrub ok 2024-09-13T15:09:19.208 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:19.234 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:18 smithi159 ceph-mon[102519]: 14.3 scrub starts 2024-09-13T15:09:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:18 smithi159 ceph-mon[102519]: 14.3 scrub ok 2024-09-13T15:09:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:18 smithi159 ceph-mon[102519]: 14.b scrub starts 2024-09-13T15:09:19.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:18 smithi159 ceph-mon[102519]: 14.b scrub ok 2024-09-13T15:09:19.336 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:19.358 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:19.460 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:19.487 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:19.589 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:19.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.612 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:19.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.738 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:19.839 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:19.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.861 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:19.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:19.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:19.985 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:19.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:19.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:20.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[113926]: pgmap v1597: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%) 2024-09-13T15:09:20.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[113926]: 14.21 scrub starts 2024-09-13T15:09:20.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[113926]: 14.21 scrub ok 2024-09-13T15:09:20.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[113926]: 14.1b scrub starts 2024-09-13T15:09:20.017 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[113926]: 14.1b scrub ok 2024-09-13T15:09:20.018 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:19] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.51.0" 2024-09-13T15:09:20.018 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[108570]: pgmap v1597: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%) 2024-09-13T15:09:20.018 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[108570]: 14.21 scrub starts 2024-09-13T15:09:20.018 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[108570]: 14.21 scrub ok 2024-09-13T15:09:20.018 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[108570]: 14.1b scrub starts 2024-09-13T15:09:20.019 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:19 smithi042 ceph-mon[108570]: 14.1b scrub ok 2024-09-13T15:09:20.087 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.108 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:20.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:19 smithi159 ceph-mon[102519]: pgmap v1597: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%) 2024-09-13T15:09:20.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:19 smithi159 ceph-mon[102519]: 14.21 scrub starts 2024-09-13T15:09:20.150 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:19 smithi159 ceph-mon[102519]: 14.21 scrub ok 2024-09-13T15:09:20.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:19 smithi159 ceph-mon[102519]: 14.1b scrub starts 2024-09-13T15:09:20.151 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:19 smithi159 ceph-mon[102519]: 14.1b scrub ok 2024-09-13T15:09:20.210 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:20.233 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:20.335 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:20.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.360 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:20.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:20.461 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:20.482 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:20.584 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:20.598 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:09:20.605 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:20.707 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:20.729 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:20.831 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:20.854 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:20.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:20.878 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.17 scrub starts 2024-09-13T15:09:20.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.17 scrub starts 2024-09-13T15:09:20.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.17 scrub ok 2024-09-13T15:09:20.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.23 scrub starts 2024-09-13T15:09:20.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.23 scrub ok 2024-09-13T15:09:20.934 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.18 scrub starts 2024-09-13T15:09:20.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: 14.18 scrub ok 2024-09-13T15:09:20.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4214514190' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:20.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: Health check update: Degraded data redundancy: 854/477 objects degraded (179.036%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:20.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: osdmap e451: 8 total, 8 up, 7 in 2024-09-13T15:09:20.935 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:20.956 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:20.978 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:20.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:21.087 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:09:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.105 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:21.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.17 scrub ok 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.23 scrub starts 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.23 scrub ok 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.18 scrub starts 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: 14.18 scrub ok 2024-09-13T15:09:21.128 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4214514190' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: Health check update: Degraded data redundancy: 854/477 objects degraded (179.036%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: osdmap e451: 8 total, 8 up, 7 in 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.17 scrub starts 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.17 scrub ok 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.23 scrub starts 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.23 scrub ok 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.18 scrub starts 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: 14.18 scrub ok 2024-09-13T15:09:21.129 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4214514190' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:21.130 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: Health check update: Degraded data redundancy: 854/477 objects degraded (179.036%), 1 pg degraded (PG_DEGRADED) 2024-09-13T15:09:21.130 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: osdmap e451: 8 total, 8 up, 7 in 2024-09-13T15:09:21.130 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:21.206 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:21.229 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:21.252 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:09:21.252 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:09:21.252 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:09:21.252 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:09:21.252 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:21.253 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:09:21.254 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:09:21.254 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:21.254 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:21.330 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.352 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:21.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.453 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:21.454 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:21.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:21.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:21.576 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:21.599 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:21.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:21.701 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.723 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.824 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:21.845 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:21.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:21.971 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:21.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:21.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.15 scrub starts 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.15 scrub ok 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.4 scrub starts 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.4 scrub ok 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: pgmap v1599: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%); 0 B/s, 113 objects/s recovering 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.1f scrub starts 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: 14.1f scrub ok 2024-09-13T15:09:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:21 smithi159 ceph-mon[102519]: osdmap e452: 8 total, 8 up, 7 in 2024-09-13T15:09:22.016 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:22.073 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:22.096 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:22.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.15 scrub starts 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.15 scrub ok 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.4 scrub starts 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.4 scrub ok 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: pgmap v1599: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%); 0 B/s, 113 objects/s recovering 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.1f scrub starts 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: 14.1f scrub ok 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[113926]: osdmap e452: 8 total, 8 up, 7 in 2024-09-13T15:09:22.150 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.15 scrub starts 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.15 scrub ok 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.4 scrub starts 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.4 scrub ok 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: pgmap v1599: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 8.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/477 objects degraded (179.036%); 0 B/s, 113 objects/s recovering 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.1f scrub starts 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: 14.1f scrub ok 2024-09-13T15:09:22.151 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:21 smithi042 ceph-mon[108570]: osdmap e452: 8 total, 8 up, 7 in 2024-09-13T15:09:22.197 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.220 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:22.321 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:22.342 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:22.444 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:22.465 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:22.567 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:22.588 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:22.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:22.712 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:22.814 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:22.836 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:22.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:22.938 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:22.959 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:22.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:23.060 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:23.081 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:23.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:22 smithi042 ceph-mon[108570]: 14.16 scrub starts 2024-09-13T15:09:23.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:22 smithi042 ceph-mon[108570]: 14.16 scrub ok 2024-09-13T15:09:23.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:22 smithi042 ceph-mon[113926]: 14.16 scrub starts 2024-09-13T15:09:23.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:22 smithi042 ceph-mon[113926]: 14.16 scrub ok 2024-09-13T15:09:23.183 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:23.204 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:22 smithi159 ceph-mon[102519]: 14.16 scrub starts 2024-09-13T15:09:23.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:22 smithi159 ceph-mon[102519]: 14.16 scrub ok 2024-09-13T15:09:23.306 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:23.327 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:23.406 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:23.429 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:23.450 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:23.552 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:23.573 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:23.674 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:23.696 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:23.798 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:23.819 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:23.921 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:23.947 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-13T15:09:23.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:23.948 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:23.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:24.041 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 5, 6, 7, 4, 1] out_osds: [0] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:09:24.041 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:09:24.041 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [2, 3, 5, 6, 7, 4, 1] 2024-09-13T15:09:24.041 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd out 2 2024-09-13T15:09:24.049 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:24.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.109 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:24.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[108570]: 14.14 scrub starts 2024-09-13T15:09:24.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[108570]: 14.14 scrub ok 2024-09-13T15:09:24.162 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[108570]: pgmap v1601: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/279 objects degraded (306.093%); 0 B/s, 98 objects/s recovering 2024-09-13T15:09:24.163 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[113926]: 14.14 scrub starts 2024-09-13T15:09:24.163 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[113926]: 14.14 scrub ok 2024-09-13T15:09:24.163 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:23 smithi042 ceph-mon[113926]: pgmap v1601: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/279 objects degraded (306.093%); 0 B/s, 98 objects/s recovering 2024-09-13T15:09:24.209 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:24.231 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:24.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:23 smithi159 ceph-mon[102519]: 14.14 scrub starts 2024-09-13T15:09:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:23 smithi159 ceph-mon[102519]: 14.14 scrub ok 2024-09-13T15:09:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:23 smithi159 ceph-mon[102519]: pgmap v1601: 119 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 854/279 objects degraded (306.093%); 0 B/s, 98 objects/s recovering 2024-09-13T15:09:24.333 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:24.358 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:24.460 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:24.483 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:24.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:24.585 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.608 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.710 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.732 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.834 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:24.857 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:24.890 INFO:teuthology.orchestra.run.smithi042.stderr:pool 'unique_pool_9' removedpool 'unique_pool_9' removed 2024-09-13T15:09:24.959 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:24.983 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:24.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:25.085 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:25.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[108570]: 14.12 scrub starts 2024-09-13T15:09:25.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[108570]: 14.12 scrub ok 2024-09-13T15:09:25.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:09:25.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2341008437' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[113926]: 14.12 scrub starts 2024-09-13T15:09:25.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[113926]: 14.12 scrub ok 2024-09-13T15:09:25.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:09:25.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2341008437' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:24 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.109 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:25.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:25.211 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:24 smithi159 ceph-mon[102519]: 14.12 scrub starts 2024-09-13T15:09:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:24 smithi159 ceph-mon[102519]: 14.12 scrub ok 2024-09-13T15:09:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-13T15:09:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:24 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2341008437' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:24 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true} : dispatch 2024-09-13T15:09:25.257 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:25.359 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:25.382 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:25.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:25.484 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:25.505 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:25.606 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:25.629 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:25.731 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.753 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:25.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:25.854 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.876 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:25.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:25.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:25.977 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:25.999 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:25.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: 14.2 scrub starts 2024-09-13T15:09:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: 14.2 scrub ok 2024-09-13T15:09:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: pgmap v1602: 119 pgs: 119 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 854/279 objects degraded (306.093%), 1 pg degraded) 2024-09-13T15:09:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:09:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:09:26.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[108570]: osdmap e453: 8 total, 8 up, 7 in 2024-09-13T15:09:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: 14.2 scrub starts 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: 14.2 scrub ok 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: pgmap v1602: 119 pgs: 119 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 854/279 objects degraded (306.093%), 1 pg degraded) 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:09:26.101 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:25 smithi042 ceph-mon[113926]: osdmap e453: 8 total, 8 up, 7 in 2024-09-13T15:09:26.102 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:26.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.125 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:26.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:26.226 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:26.247 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:26.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: 14.2 scrub starts 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: 14.2 scrub ok 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: pgmap v1602: 119 pgs: 119 active+clean; 4.8 MiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 854/279 objects degraded (306.093%), 1 pg degraded) 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_9", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_9", "pool2": "unique_pool_9", "yes_i_really_really_mean_it": true}]': finished 2024-09-13T15:09:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:25 smithi159 ceph-mon[102519]: osdmap e453: 8 total, 8 up, 7 in 2024-09-13T15:09:26.349 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:26.370 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:26.472 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.498 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:26.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:26.600 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:26.621 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:26.653 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T15:09:26.653 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-13T15:09:26.653 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T15:09:26.660 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-13T15:09:26.660 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T15:09:26.661 DEBUG:teuthology.orchestra.run.smithi042:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:09:26.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.687 INFO:teuthology.orchestra.run.smithi042.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T15:09:26.687 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T15:09:26.723 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:26.744 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:26.774 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T15:09:26.775 DEBUG:teuthology.orchestra.run.smithi042:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T15:09:26.791 INFO:tasks.workunit:timeout=3h 2024-09-13T15:09:26.791 INFO:tasks.workunit:cleanup=True 2024-09-13T15:09:26.791 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph-ci.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:09:26.846 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:26.880 INFO:tasks.workunit.client.0.smithi042.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T15:09:26.899 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:26.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:26.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:27.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.023 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:27.091 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:26 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:27.091 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:26 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:27.125 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:27.147 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:27.186 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:27.249 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:27.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:26 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:09:27.273 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:27.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:27.398 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:27.500 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:27.530 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:27.632 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.656 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:27.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:27.780 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:27.882 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:27.904 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:27.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:28.006 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:28.028 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:28.130 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:28.151 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:27 smithi159 ceph-mon[102519]: pgmap v1604: 83 pgs: 83 active+clean; 582 KiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:27 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.253 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:28.275 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:28.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:28.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[113926]: pgmap v1604: 83 pgs: 83 active+clean; 582 KiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[108570]: pgmap v1604: 83 pgs: 83 active+clean; 582 KiB data, 9.7 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.307 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.308 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:27 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:28.377 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:28.400 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:28.502 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:28.506 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:09:28.506 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:09:28.506 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:09:28.506 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:09:28.506 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:09:28.513 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:28.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.526 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:28.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:28.627 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:28.649 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:28.751 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:28.772 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:28.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:28.897 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:28.920 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:28.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:28.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:28.951 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:09:29.022 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.044 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:29.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:29.146 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:29.168 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:29.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:28 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2826916862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:28 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:29.292 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:29.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:28 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2826916862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.296 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:28 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.297 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:28 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2826916862' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.297 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:28 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:09:29.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:09:29.394 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.426 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:29.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:29.527 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.551 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:29.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:29.653 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:29.676 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:29.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:29.778 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:29.799 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:29.825 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:29] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.51.0" 2024-09-13T15:09:29.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:29.925 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:29.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:30.026 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:30.049 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:30.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[108570]: pgmap v1605: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:09:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:09:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:30.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[108570]: osdmap e454: 8 total, 8 up, 7 in 2024-09-13T15:09:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[113926]: pgmap v1605: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:09:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:09:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:30.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:29 smithi042 ceph-mon[113926]: osdmap e454: 8 total, 8 up, 7 in 2024-09-13T15:09:30.152 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:30.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.176 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:30.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:29 smithi159 ceph-mon[102519]: pgmap v1605: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:29 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-13T15:09:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:29 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:09:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:29 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:09:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:29 smithi159 ceph-mon[102519]: osdmap e454: 8 total, 8 up, 7 in 2024-09-13T15:09:30.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:30.299 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:30.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:30.401 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:30.422 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:30.524 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:30.546 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:30.648 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:30.670 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:30.772 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:30.797 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:30.899 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:30.920 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:30.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:31.022 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:31.044 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:31.145 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:31.167 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:31.270 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.295 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:31.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:31.396 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:31.423 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:31.525 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:31.547 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:31.648 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:31.669 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:31.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:31.771 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:31.793 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:31.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:31.894 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:31.918 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:31.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:32.019 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:32.040 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:32.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:32.142 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:32.163 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:32.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:31 smithi159 ceph-mon[102519]: pgmap v1607: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:31 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.265 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:32.286 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:32.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[108570]: pgmap v1607: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[113926]: pgmap v1607: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:32.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:31 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:32.380 DEBUG:teuthology.orchestra.run.smithi042:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T15:09:32.388 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:32.448 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:32.471 INFO:tasks.workunit.client.0.smithi042.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T15:09:32.474 INFO:tasks.workunit.client.0.smithi042.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:09:32.474 INFO:tasks.workunit.client.0.smithi042.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T15:09:32.550 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:32.573 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:32.675 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:32.697 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:32.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:32.799 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:32.821 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:32.923 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:32.944 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:32.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:32.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:33.046 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:33.069 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:33.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:33.171 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:33.194 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:33.296 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:33.319 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:33.420 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:33.444 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:33.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:33.545 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.568 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:33.670 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:33.692 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:33.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:09:33.721 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:09:33.722 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:09:33.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:33.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:33.722 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:33.729 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:33.739 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:09:33.739 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:33.760 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:09:33.760 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:33.778 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:09:33.778 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:33.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:33.795 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:09:33.795 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:33.811 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:09:33.811 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:33.817 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:33.919 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:33.942 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:33.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:34.044 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:34.066 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:34.168 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:34.190 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:34.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:34.224 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:33 smithi042 ceph-mon[113926]: pgmap v1608: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:34.224 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3354880898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:34.224 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:33 smithi042 ceph-mon[108570]: pgmap v1608: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:34.225 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3354880898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:33 smithi159 ceph-mon[102519]: pgmap v1608: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3354880898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:34.291 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:34.315 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:34.417 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:34.441 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:34.544 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.569 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:34.671 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:34.697 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:34.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:34.799 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:34.822 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:34.924 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:34.947 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:34.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:34.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:35.049 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:35.053 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-13T15:09:35.074 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:35.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:35.176 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.183 INFO:tasks.workunit.client.0.smithi042.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T15:09:35.202 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:35.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:35.304 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:35.307 INFO:tasks.workunit.client.0.smithi042.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T15:09:35.327 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:35.349 INFO:tasks.workunit.client.0.smithi042.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T15:09:35.351 INFO:tasks.workunit.client.0.smithi042.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:09:35.351 INFO:tasks.workunit.client.0.smithi042.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T15:09:35.391 INFO:tasks.workunit.client.0.smithi042.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T15:09:35.395 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T15:09:35.395 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T15:09:35.429 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.458 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:35.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:35.487 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-13T15:09:35.489 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-13T15:09:35.489 DEBUG:teuthology.orchestra.run.smithi042:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-13T15:09:35.559 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.587 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:35.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:35.688 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:35.714 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:35.801 INFO:tasks.workunit.client.0.smithi042.stdout:Running main() from gmock_main.cc 2024-09-13T15:09:35.801 INFO:tasks.workunit.client.0.smithi042.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-13T15:09:35.801 INFO:tasks.workunit.client.0.smithi042.stdout:[----------] Global test environment set-up. 2024-09-13T15:09:35.801 INFO:tasks.workunit.client.0.smithi042.stdout:[----------] 70 tests from TestClsRbd 2024-09-13T15:09:35.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:35.839 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:35.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:35.941 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:35.966 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:35.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:35.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:36.068 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:36.093 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:36.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:36.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:36.218 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:35 smithi159 ceph-mon[102519]: pgmap v1609: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:36.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:36.320 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:36.346 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[108570]: pgmap v1609: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:36.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[113926]: pgmap v1609: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:36.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:36.448 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:36.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.474 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:36.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:36.575 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:36.598 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:36.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:36.700 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:36.725 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:36.827 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:36.849 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:36.951 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:36.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:36.977 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:36.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:37.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.101 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:37.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:37.195 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:09:37.195 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:09:37.204 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.230 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:37.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:37.331 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:37.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:09:37.128+0000 7fcad2a32640 -1 mon.a@0(leader).osd e455 definitely_dead 0 2024-09-13T15:09:37.356 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:37.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:37.458 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:37.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.485 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:37.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:37.586 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:37.611 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:37.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:37.713 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:37.739 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:37.818 INFO:teuthology.orchestra.run.smithi042.stderr:marked out osd.2. 2024-09-13T15:09:37.819 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: pgmap v1610: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:37.820 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: osdmap e455: 8 total, 8 up, 7 in 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3782492781' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[108570]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: pgmap v1610: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: osdmap e455: 8 total, 8 up, 7 in 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:37.821 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3782492781' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:37.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:37.822 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:37 smithi042 ceph-mon[113926]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:09:37.822 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-13T15:09:37.840 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:37.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.866 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:37.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:37.967 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:37.991 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:37.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:37.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:38.000 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-13T15:09:38.093 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.117 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:38.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:38.218 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:38.223 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:38.243 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:38.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: pgmap v1610: 83 pgs: 83 active+clean; 582 KiB data, 9.6 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: osdmap e455: 8 total, 8 up, 7 in 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3782492781' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-09-13T15:09:38.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:37 smithi159 ceph-mon[102519]: Client client.admin marked osd.2 out, while it was still marked up 2024-09-13T15:09:38.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.367 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:38.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:38.439 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-13T15:09:38.469 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:38.495 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:38.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:38.597 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:38.622 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:38.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:38.724 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:38.750 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:38.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:38.850 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_all_features (1030 ms) 2024-09-13T15:09:38.850 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.copyup 2024-09-13T15:09:38.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:38.877 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:38.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:38.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:38.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:09:38.937 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[113926]: osdmap e456: 8 total, 8 up, 6 in 2024-09-13T15:09:38.938 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:38.938 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:09:38.938 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:38 smithi042 ceph-mon[108570]: osdmap e456: 8 total, 8 up, 6 in 2024-09-13T15:09:38.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:39.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.004 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:39.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:39.036 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.copyup (186 ms) 2024-09-13T15:09:39.036 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-13T15:09:39.066 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.sparse_copyup (30 ms) 2024-09-13T15:09:39.066 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-13T15:09:39.073 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-09-13T15:09:39.073 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-13T15:09:39.100 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.add_remove_child (27 ms) 2024-09-13T15:09:39.100 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-13T15:09:39.105 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:39.125 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2024-09-13T15:09:39.126 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.create 2024-09-13T15:09:39.127 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:39.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:39.144 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.create (18 ms) 2024-09-13T15:09:39.145 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_features 2024-09-13T15:09:39.145 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-13T15:09:39.145 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-13T15:09:39.147 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-13T15:09:39.147 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-13T15:09:39.148 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-13T15:09:39.148 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-13T15:09:39.149 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-13T15:09:39.149 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-13T15:09:39.150 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-13T15:09:39.150 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-13T15:09:39.153 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-13T15:09:39.154 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.get_size 2024-09-13T15:09:39.157 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-13T15:09:39.157 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.set_size 2024-09-13T15:09:39.161 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-13T15:09:39.161 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-13T15:09:39.180 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.protection_status (18 ms) 2024-09-13T15:09:39.180 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-13T15:09:39.187 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-13T15:09:39.187 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-13T15:09:39.225 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.parents_v1 (38 ms) 2024-09-13T15:09:39.225 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-13T15:09:39.229 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:38 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3135534154' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi042-346998-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T15:09:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:38 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-09-13T15:09:39.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:38 smithi159 ceph-mon[102519]: osdmap e456: 8 total, 8 up, 6 in 2024-09-13T15:09:39.251 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2024-09-13T15:09:39.251 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-13T15:09:39.252 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:39.273 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2024-09-13T15:09:39.273 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-13T15:09:39.277 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2024-09-13T15:09:39.277 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-13T15:09:39.285 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-13T15:09:39.286 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-13T15:09:39.286 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2024-09-13T15:09:39.287 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-13T15:09:39.293 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_resize (7 ms) 2024-09-13T15:09:39.293 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-13T15:09:39.299 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2024-09-13T15:09:39.300 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-13T15:09:39.300 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-13T15:09:39.300 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-13T15:09:39.308 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_snap_add (9 ms) 2024-09-13T15:09:39.308 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-13T15:09:39.315 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.object_map_snap_remove (7 ms) 2024-09-13T15:09:39.315 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.flags 2024-09-13T15:09:39.320 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-13T15:09:39.320 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.metadata 2024-09-13T15:09:39.342 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.metadata (22 ms) 2024-09-13T15:09:39.342 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.set_features 2024-09-13T15:09:39.349 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-13T15:09:39.349 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror 2024-09-13T15:09:39.354 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:39.376 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:39.384 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2024-09-13T15:09:39.384 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-13T15:09:39.398 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror_image (14 ms) 2024-09-13T15:09:39.398 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-13T15:09:39.477 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:39.503 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:39.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:39.605 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:39.630 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:39.732 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:39.758 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:39.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:39.885 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:39.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:39.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:40.011 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[108570]: pgmap v1613: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T15:09:40.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:40.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[108570]: osdmap e457: 8 total, 8 up, 6 in 2024-09-13T15:09:40.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:39] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.51.0" 2024-09-13T15:09:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[113926]: pgmap v1613: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T15:09:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:40.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:39 smithi042 ceph-mon[113926]: osdmap e457: 8 total, 8 up, 6 in 2024-09-13T15:09:40.114 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:40.135 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:40.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:40.237 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:39 smithi159 ceph-mon[102519]: pgmap v1613: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-09-13T15:09:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:39 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:40.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:39 smithi159 ceph-mon[102519]: osdmap e457: 8 total, 8 up, 6 in 2024-09-13T15:09:40.263 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:40.365 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:40.390 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:40.492 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:40.518 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:40.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:40.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.648 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:40.749 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.777 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:40.878 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:40.903 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:40.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:40.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:41.005 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:41.030 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:40 smithi042 ceph-mon[113926]: osdmap e458: 8 total, 8 up, 6 in 2024-09-13T15:09:41.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:40 smithi042 ceph-mon[113926]: osdmap e459: 8 total, 8 up, 6 in 2024-09-13T15:09:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:40 smithi042 ceph-mon[108570]: osdmap e458: 8 total, 8 up, 6 in 2024-09-13T15:09:41.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:40 smithi042 ceph-mon[108570]: osdmap e459: 8 total, 8 up, 6 in 2024-09-13T15:09:41.132 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:41.153 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:40 smithi159 ceph-mon[102519]: osdmap e458: 8 total, 8 up, 6 in 2024-09-13T15:09:41.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:40 smithi159 ceph-mon[102519]: osdmap e459: 8 total, 8 up, 6 in 2024-09-13T15:09:41.255 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.278 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:41.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:41.379 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:41.407 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:41.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:41.509 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:41.535 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:41.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:41.637 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:41.652 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:41.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.663 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:41.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:41.677 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:41.764 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:41.785 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:41.887 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:41.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:41.909 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:41.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:41.910 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror_image_status (2513 ms) 2024-09-13T15:09:41.910 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-13T15:09:41.998 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror_image_map (88 ms) 2024-09-13T15:09:41.999 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-13T15:09:42.006 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-13T15:09:42.006 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-13T15:09:42.010 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:42.018 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2024-09-13T15:09:42.019 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-13T15:09:42.021 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2024-09-13T15:09:42.021 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-13T15:09:42.023 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-13T15:09:42.023 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-13T15:09:42.026 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-13T15:09:42.027 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-13T15:09:42.033 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.033 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2024-09-13T15:09:42.033 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-13T15:09:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:42.036 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-13T15:09:42.036 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-13T15:09:42.040 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-13T15:09:42.040 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-13T15:09:42.042 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-13T15:09:42.043 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-13T15:09:42.045 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-13T15:09:42.046 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-13T15:09:42.048 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-13T15:09:42.048 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-13T15:09:42.053 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-13T15:09:42.053 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-13T15:09:42.055 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-09-13T15:09:42.055 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-13T15:09:42.058 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-13T15:09:42.058 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-13T15:09:42.060 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-13T15:09:42.060 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-13T15:09:42.062 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (1 ms) 2024-09-13T15:09:42.062 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-13T15:09:42.064 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (3 ms) 2024-09-13T15:09:42.064 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-13T15:09:42.067 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-09-13T15:09:42.067 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-13T15:09:42.070 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-13T15:09:42.070 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-843-g2c01458a/rpm/el9/BUILD/ceph-18.2.4-843-g2c01458a/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout:Expected equality of these values: 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout: 1U 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout: Which is: 1 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout: keys.size() 2024-09-13T15:09:42.072 INFO:tasks.workunit.client.0.smithi042.stdout: Which is: 3 2024-09-13T15:09:42.073 INFO:tasks.workunit.client.0.smithi042.stdout:[ FAILED ] TestClsRbd.group_snap_set (3 ms) 2024-09-13T15:09:42.073 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-13T15:09:42.075 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-13T15:09:42.076 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-13T15:09:42.092 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-13T15:09:42.092 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-13T15:09:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:41 smithi042 ceph-mon[113926]: pgmap v1617: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:09:42.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:41 smithi042 ceph-mon[113926]: osdmap e460: 8 total, 8 up, 6 in 2024-09-13T15:09:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:41 smithi042 ceph-mon[108570]: pgmap v1617: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:09:42.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:41 smithi042 ceph-mon[108570]: osdmap e460: 8 total, 8 up, 6 in 2024-09-13T15:09:42.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:42.157 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:41 smithi159 ceph-mon[102519]: pgmap v1617: 115 pgs: 11 peering, 23 creating+peering, 7 unknown, 74 active+clean; 378 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 13 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-13T15:09:42.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:41 smithi159 ceph-mon[102519]: osdmap e460: 8 total, 8 up, 6 in 2024-09-13T15:09:42.259 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.283 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:42.288 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (197 ms) 2024-09-13T15:09:42.288 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-13T15:09:42.291 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-09-13T15:09:42.292 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-13T15:09:42.293 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-13T15:09:42.293 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-13T15:09:42.301 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-13T15:09:42.301 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.op_features 2024-09-13T15:09:42.307 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-13T15:09:42.307 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-13T15:09:42.327 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.clone_parent (19 ms) 2024-09-13T15:09:42.327 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-13T15:09:42.335 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2024-09-13T15:09:42.335 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-13T15:09:42.345 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.clone_child (10 ms) 2024-09-13T15:09:42.345 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-13T15:09:42.352 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-13T15:09:42.352 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.migration 2024-09-13T15:09:42.362 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-09-13T15:09:42.362 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-13T15:09:42.370 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.migration_v1 (9 ms) 2024-09-13T15:09:42.370 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-13T15:09:42.384 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.408 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:42.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:42.509 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:42.530 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:42.632 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:42.653 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:42.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3 2024-09-13T15:09:42.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.779 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:42.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.3' 2024-09-13T15:09:42.829 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is set 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1950261501' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[113926]: 1.8 scrub starts 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[113926]: 1.8 scrub ok 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1950261501' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[108570]: 1.8 scrub starts 2024-09-13T15:09:42.843 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:42 smithi042 ceph-mon[108570]: 1.8 scrub ok 2024-09-13T15:09:42.880 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:42.902 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:42.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:42.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.004 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:43.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.028 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:43.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:43.129 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.150 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:43.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1950261501' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:42 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-13T15:09:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:42 smithi159 ceph-mon[102519]: 1.8 scrub starts 2024-09-13T15:09:43.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:42 smithi159 ceph-mon[102519]: 1.8 scrub ok 2024-09-13T15:09:43.253 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:43.274 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:43.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:43.375 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1 2024-09-13T15:09:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.400 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:43.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.1' 2024-09-13T15:09:43.501 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:43.523 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:43.625 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:43.649 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:43.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:43.683 INFO:tasks.thrashosds.thrasher:in_osds: [3, 5, 6, 7, 4, 1] out_osds: [0, 2] dead_osds: [] live_osds: [1, 2, 3, 4, 5, 6, 7, 0] 2024-09-13T15:09:43.683 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-13T15:09:43.684 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-09-13T15:09:43.684 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd in 2 2024-09-13T15:09:43.751 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:43.773 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:43.832 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1461 ms) 2024-09-13T15:09:43.832 INFO:tasks.workunit.client.0.smithi042.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-13T15:09:43.866 INFO:tasks.workunit.client.0.smithi042.stdout:[ OK ] TestClsRbd.sparsify (35 ms) 2024-09-13T15:09:43.875 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:43.896 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:43.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:43.998 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:44.022 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:44.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:44.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: 1.a scrub starts 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: 1.a scrub ok 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: pgmap v1619: 115 pgs: 11 peering, 3 creating+peering, 2 unknown, 99 active+clean; 483 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 211 op/s; 79 KiB/s, 2 keys/s, 0 objects/s recovering 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[108570]: osdmap e461: 8 total, 8 up, 6 in 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: 1.a scrub starts 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: 1.a scrub ok 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: pgmap v1619: 115 pgs: 11 peering, 3 creating+peering, 2 unknown, 99 active+clean; 483 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 211 op/s; 79 KiB/s, 2 keys/s, 0 objects/s recovering 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:44.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:43 smithi042 ceph-mon[113926]: osdmap e461: 8 total, 8 up, 6 in 2024-09-13T15:09:44.124 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:44.146 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:44.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:44.248 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4 2024-09-13T15:09:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: 1.a scrub starts 2024-09-13T15:09:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: 1.a scrub ok 2024-09-13T15:09:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: pgmap v1619: 115 pgs: 11 peering, 3 creating+peering, 2 unknown, 99 active+clean; 483 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 211 op/s; 79 KiB/s, 2 keys/s, 0 objects/s recovering 2024-09-13T15:09:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-13T15:09:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-13T15:09:44.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:43 smithi159 ceph-mon[102519]: osdmap e461: 8 total, 8 up, 6 in 2024-09-13T15:09:44.270 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:44.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.4' 2024-09-13T15:09:44.372 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5 2024-09-13T15:09:44.394 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.5' 2024-09-13T15:09:44.496 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6 2024-09-13T15:09:44.518 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:44.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.6' 2024-09-13T15:09:44.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0 2024-09-13T15:09:44.643 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.0' 2024-09-13T15:09:44.745 DEBUG:teuthology.orchestra.run.smithi159:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7 2024-09-13T15:09:44.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.768 INFO:teuthology.orchestra.run.smithi159.stderr:sudo: docker: command not found 2024-09-13T15:09:44.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi159 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.7' 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[----------] 70 tests from TestClsRbd (6047 ms total) 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout: 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[----------] Global test environment tear-down 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[==========] 70 tests from 1 test suite ran. (9037 ms total) 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[ PASSED ] 69 tests. 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[ FAILED ] 1 test, listed below: 2024-09-13T15:09:44.838 INFO:tasks.workunit.client.0.smithi042.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-13T15:09:44.839 INFO:tasks.workunit.client.0.smithi042.stdout: 2024-09-13T15:09:44.839 INFO:tasks.workunit.client.0.smithi042.stdout: 1 FAILED TEST 2024-09-13T15:09:44.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.861 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-13T15:09:44.861 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T15:09:44.869 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:44.926 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:44.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:45.028 DEBUG:teuthology.orchestra.run.smithi042:> sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2 2024-09-13T15:09:45.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:09:45.052 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: docker: command not found 2024-09-13T15:09:45.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi042 with status 1: 'sudo docker kill -s 1 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd.2' 2024-09-13T15:09:45.091 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:44 smithi042 ceph-mon[113926]: osdmap e462: 8 total, 8 up, 6 in 2024-09-13T15:09:45.092 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:44 smithi042 ceph-mon[108570]: osdmap e462: 8 total, 8 up, 6 in 2024-09-13T15:09:45.093 INFO:tasks.thrashosds:joining thrashosds 2024-09-13T15:09:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:44 smithi159 ceph-mon[102519]: osdmap e462: 8 total, 8 up, 6 in 2024-09-13T15:09:45.795 INFO:teuthology.orchestra.run.smithi042.stdout:ERROR: (22) Invalid argument 2024-09-13T15:09:45.796 INFO:teuthology.orchestra.run.smithi042.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-13T15:09:46.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[113926]: pgmap v1622: 115 pgs: 115 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 3.31k op/s; 38 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:09:46.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:09:46.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[113926]: osdmap e463: 8 total, 8 up, 6 in 2024-09-13T15:09:46.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[108570]: pgmap v1622: 115 pgs: 115 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 3.31k op/s; 38 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:09:46.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:09:46.101 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:45 smithi042 ceph-mon[108570]: osdmap e463: 8 total, 8 up, 6 in 2024-09-13T15:09:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:45 smithi159 ceph-mon[102519]: pgmap v1622: 115 pgs: 115 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 4.1 MiB/s rd, 2.0 MiB/s wr, 3.31k op/s; 38 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:09:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:45 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:09:46.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:45 smithi159 ceph-mon[102519]: osdmap e463: 8 total, 8 up, 6 in 2024-09-13T15:09:47.193 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-13T15:09:47.750 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5: {} 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-13T15:09:47.751 INFO:teuthology.orchestra.run.smithi042.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-13T15:09:47.764 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6: {} 2024-09-13T15:09:47.764 INFO:teuthology.orchestra.run.smithi042.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-13T15:09:47.782 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7: {} 2024-09-13T15:09:47.782 INFO:teuthology.orchestra.run.smithi042.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-13T15:09:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:47 smithi042 ceph-mon[113926]: pgmap v1624: 83 pgs: 83 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 14 KiB/s rd, 43 KiB/s wr, 10 op/s; 31 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-13T15:09:48.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:47 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/333530649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:47 smithi042 ceph-mon[108570]: pgmap v1624: 83 pgs: 83 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 14 KiB/s rd, 43 KiB/s wr, 10 op/s; 31 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-13T15:09:48.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:47 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/333530649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:48.142 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:47 smithi159 ceph-mon[102519]: pgmap v1624: 83 pgs: 83 active+clean; 687 KiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 14 KiB/s rd, 43 KiB/s wr, 10 op/s; 31 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-13T15:09:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:47 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/333530649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-13T15:09:48.667 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-13T15:09:50.055 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:49 smithi042 ceph-mon[113926]: pgmap v1625: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:50.055 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:49] "GET /metrics HTTP/1.1" 200 34134 "" "Prometheus/2.51.0" 2024-09-13T15:09:50.056 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:49 smithi042 ceph-mon[108570]: pgmap v1625: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:50.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:49 smithi159 ceph-mon[102519]: pgmap v1625: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:52.030 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[108570]: pgmap v1626: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[113926]: pgmap v1626: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:52.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:51 smithi159 ceph-mon[102519]: pgmap v1626: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:09:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:52.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:09:52.809 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:52 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:09:52.450+0000 7fcad2a32640 -1 mon.a@0(leader).osd e463 definitely_dead 0 2024-09-13T15:09:52.818 INFO:teuthology.orchestra.run.smithi042.stderr:marked in osd.2. 2024-09-13T15:09:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1828508236' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:52 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1828508236' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:52 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1828508236' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:52 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-09-13T15:09:53.646 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-09-13T15:09:54.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[108570]: pgmap v1627: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2024-09-13T15:09:54.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:09:54.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[108570]: osdmap e464: 8 total, 8 up, 7 in 2024-09-13T15:09:54.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[113926]: pgmap v1627: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2024-09-13T15:09:54.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:09:54.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:53 smithi042 ceph-mon[113926]: osdmap e464: 8 total, 8 up, 7 in 2024-09-13T15:09:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:53 smithi159 ceph-mon[102519]: pgmap v1627: 83 pgs: 83 active+clean; 634 KiB data, 8.4 GiB used, 528 GiB / 536 GiB avail 2024-09-13T15:09:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:53 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-09-13T15:09:54.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:53 smithi159 ceph-mon[102519]: osdmap e464: 8 total, 8 up, 7 in 2024-09-13T15:09:55.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[113926]: osdmap e465: 8 total, 8 up, 7 in 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[113926]: osdmap e466: 8 total, 8 up, 7 in 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[108570]: osdmap e465: 8 total, 8 up, 7 in 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:54 smithi042 ceph-mon[108570]: osdmap e466: 8 total, 8 up, 7 in 2024-09-13T15:09:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:54 smithi159 ceph-mon[102519]: osdmap e465: 8 total, 8 up, 7 in 2024-09-13T15:09:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:54 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:55.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:54 smithi159 ceph-mon[102519]: osdmap e466: 8 total, 8 up, 7 in 2024-09-13T15:09:55.625 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:56.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: pgmap v1630: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: osdmap e467: 8 total, 8 up, 7 in 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/4050196370' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.089 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: pgmap v1630: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: osdmap e467: 8 total, 8 up, 7 in 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/4050196370' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.090 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:55 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: pgmap v1630: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T15:09:56.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:09:56.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:09:56.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:09:56.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: osdmap e467: 8 total, 8 up, 7 in 2024-09-13T15:09:56.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/4050196370' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:55 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-13T15:09:56.642 INFO:teuthology.orchestra.run.smithi042.stderr:noscrub is unset 2024-09-13T15:09:57.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[113926]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T15:09:57.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:57.087 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[113926]: osdmap e468: 8 total, 8 up, 7 in 2024-09-13T15:09:57.087 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[108570]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T15:09:57.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:57.088 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:56 smithi042 ceph-mon[108570]: osdmap e468: 8 total, 8 up, 7 in 2024-09-13T15:09:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:56 smithi159 ceph-mon[102519]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-09-13T15:09:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:56 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-13T15:09:57.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:56 smithi159 ceph-mon[102519]: osdmap e468: 8 total, 8 up, 7 in 2024-09-13T15:09:58.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[108570]: pgmap v1634: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:58.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[108570]: 1.a scrub starts 2024-09-13T15:09:58.060 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[108570]: 1.a scrub ok 2024-09-13T15:09:58.060 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[113926]: pgmap v1634: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:58.060 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[113926]: 1.a scrub starts 2024-09-13T15:09:58.061 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:57 smithi042 ceph-mon[113926]: 1.a scrub ok 2024-09-13T15:09:58.081 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-13T15:09:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:57 smithi159 ceph-mon[102519]: pgmap v1634: 83 pgs: 14 peering, 3 unknown, 66 active+clean; 325 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail 2024-09-13T15:09:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:57 smithi159 ceph-mon[102519]: 1.a scrub starts 2024-09-13T15:09:58.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:57 smithi159 ceph-mon[102519]: 1.a scrub ok 2024-09-13T15:09:58.302 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:09:58.302 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:09:58.303 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:09:58.303 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:09:58.303 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:09:58.647 INFO:tasks.thrashosds.thrasher:inning osd 2024-09-13T15:09:58.647 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-09-13T15:09:58.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd in 0 2024-09-13T15:09:58.940 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:58 smithi042 ceph-mon[113926]: 1.1 scrub starts 2024-09-13T15:09:58.941 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:58 smithi042 ceph-mon[113926]: 1.1 scrub ok 2024-09-13T15:09:58.941 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:58 smithi042 ceph-mon[108570]: 1.1 scrub starts 2024-09-13T15:09:58.941 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:58 smithi042 ceph-mon[108570]: 1.1 scrub ok 2024-09-13T15:09:58.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-13T15:09:59.040 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:59.058 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:09:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:58 smithi159 ceph-mon[102519]: 1.1 scrub starts 2024-09-13T15:09:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:58 smithi159 ceph-mon[102519]: 1.1 scrub ok 2024-09-13T15:10:00.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[108570]: pgmap v1635: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 52 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:10:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[108570]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:10:00.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[108570]: Cluster is now healthy 2024-09-13T15:10:00.099 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:09:59] "GET /metrics HTTP/1.1" 200 34140 "" "Prometheus/2.51.0" 2024-09-13T15:10:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[113926]: pgmap v1635: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 52 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:10:00.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[113926]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:10:00.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:09:59 smithi042 ceph-mon[113926]: Cluster is now healthy 2024-09-13T15:10:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:59 smithi159 ceph-mon[102519]: pgmap v1635: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 52 KiB/s, 2 keys/s, 1 objects/s recovering 2024-09-13T15:10:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:59 smithi159 ceph-mon[102519]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-09-13T15:10:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:09:59 smithi159 ceph-mon[102519]: Cluster is now healthy 2024-09-13T15:10:01.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:00 smithi042 ceph-mon[108570]: overall HEALTH_OK 2024-09-13T15:10:01.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:00 smithi042 ceph-mon[113926]: overall HEALTH_OK 2024-09-13T15:10:01.154 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:01.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:00 smithi159 ceph-mon[102519]: overall HEALTH_OK 2024-09-13T15:10:01.960 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:01 smithi042 ceph-mon[113926]: pgmap v1636: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 43 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:01.960 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:01 smithi042 ceph-mon[108570]: pgmap v1636: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 43 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:02.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:01 smithi159 ceph-mon[102519]: pgmap v1636: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 43 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:03.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:10:03.023+0000 7fcad2a32640 -1 mon.a@0(leader).osd e468 definitely_dead 0 2024-09-13T15:10:03.843 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:03 smithi159 ceph-mon[102519]: pgmap v1637: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:03.843 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1090039695' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:03.843 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:03 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:03.843 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:10:03.845 INFO:teuthology.orchestra.run.smithi042.stderr:nodeep-scrub is unset 2024-09-13T15:10:03.852 INFO:teuthology.orchestra.run.smithi042.stderr:marked in osd.0. 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[108570]: pgmap v1637: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1090039695' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[113926]: pgmap v1637: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 32 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1090039695' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-09-13T15:10:04.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-13T15:10:05.160 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T15:10:05.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:10:05.161 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[108570]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T15:10:05.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T15:10:05.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:10:05.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:04 smithi042 ceph-mon[113926]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T15:10:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:04 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-09-13T15:10:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2254062483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-13T15:10:05.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:04 smithi159 ceph-mon[102519]: osdmap e469: 8 total, 8 up, 8 in 2024-09-13T15:10:05.538 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-09-13T15:10:05.538 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[113926]: pgmap v1639: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 31 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[113926]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[108570]: pgmap v1639: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 31 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:06.152 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[108570]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T15:10:06.153 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:10:06.153 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:05 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:06.175 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:06.175 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:10:06.175 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:10:06.175 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:10:06.175 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:110723 13.0 13:0dfe51fb:::benchmark_data_smithi042_286876_object110722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:19.600092+0000", 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 466.54997297400001, 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.31292731099999999, 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:06.176 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 110723 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600092+0000", 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600092+0000", 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600094+0000", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6530000000000001e-06 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600120+0000", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6098e-05 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600149+0000", 2024-09-13T15:10:06.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8535e-05 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600157+0000", 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.49e-06 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600229+0000", 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2284000000000001e-05 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600446+0000", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021631499999999999 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.600647+0000", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00020158200000000001 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.604203+0000", 2024-09-13T15:10:06.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003555725 2024-09-13T15:10:06.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.604369+0000", 2024-09-13T15:10:06.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000166281 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.912944+0000", 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.30857456500000002 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.912986+0000", 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1795000000000002e-05 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.913020+0000", 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3988000000000001e-05 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.185 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:110733 13.0 13:029b5a60:::benchmark_data_smithi042_286876_object110732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:02:19.625846+0000", 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 466.52421940199997, 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.28759982000000001, 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:06.186 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 110733 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625846+0000", 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625846+0000", 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625847+0000", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1060000000000001e-06 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625880+0000", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2659e-05 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625893+0000", 2024-09-13T15:10:06.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3861e-05 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625902+0000", 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5059999999999995e-06 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.625970+0000", 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8429000000000005e-05 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.626135+0000", 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000164706 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,7", 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.626297+0000", 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016234600000000001 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.629867+0000", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035695800000000001 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.630321+0000", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00045361000000000002 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.913333+0000", 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.28301248000000001 2024-09-13T15:10:06.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.913401+0000", 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7578999999999998e-05 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:02:19.913446+0000", 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4957999999999999e-05 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.192 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:75121 14.8 14:1478b5a2:::benchmark_data_smithi042_319156_object75120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:06:04.495677+0000", 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 241.65438844100001, 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.29270412400000001, 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 75121 2024-09-13T15:10:06.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495677+0000", 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495677+0000", 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495679+0000", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.911e-06 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495703+0000", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3802e-05 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495716+0000", 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3115999999999999e-05 2024-09-13T15:10:06.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495723+0000", 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7829999999999999e-06 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495805+0000", 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1695999999999998e-05 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.495961+0000", 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015559999999999999 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.496090+0000", 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012898400000000001 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.499369+0000", 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032790269999999999 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.502570+0000", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032013710000000002 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.788253+0000", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.28568250499999998 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.788333+0000", 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0085000000000003e-05 2024-09-13T15:10:06.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.788381+0000", 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8244000000000001e-05 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:75131 14.8 14:1f684b8c:::benchmark_data_smithi042_319156_object75130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:06:04.504438+0000", 2024-09-13T15:10:06.199 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 241.64562699499999, 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.29748525100000001, 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 75131 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.200 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504438+0000", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504438+0000", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504440+0000", 2024-09-13T15:10:06.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.485e-06 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504462+0000", 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1858999999999999e-05 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504485+0000", 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3153999999999999e-05 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504493+0000", 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2169999999999994e-06 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504627+0000", 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000134027 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504775+0000", 2024-09-13T15:10:06.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014768800000000001 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,3", 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.504927+0000", 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015247899999999999 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.507667+0000", 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027395929999999998 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.508178+0000", 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00051073999999999998 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.801803+0000", 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.29362542699999999 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.801882+0000", 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8646999999999994e-05 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:06:04.801924+0000", 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1934999999999997e-05 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171298 14.a 14:555b3e19:::benchmark_data_smithi042_319156_object47341:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920245+0000", 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.22982016899999, 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0296680039999999, 2024-09-13T15:10:06.206 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171298 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920245+0000", 2024-09-13T15:10:06.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920245+0000", 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920247+0000", 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8339999999999999e-06 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920258+0000", 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0801999999999999e-05 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920271+0000", 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3451e-05 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920281+0000", 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0137e-05 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.923985+0000", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037031899999999999 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.923991+0000", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3980000000000004e-06 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.949838+0000", 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.025846713 2024-09-13T15:10:06.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.949913+0000", 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5479e-05 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171312 14.a 14:5de3a2e5:::benchmark_data_smithi042_319156_object47355:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920295+0000", 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.229770272, 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1307468549999999, 2024-09-13T15:10:06.211 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171312 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920295+0000", 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920295+0000", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920296+0000", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4099999999999999e-07 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920298+0000", 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7310000000000002e-06 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920306+0000", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.469e-06 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920312+0000", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3180000000000003e-06 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924004+0000", 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036916570000000001 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924007+0000", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5899999999999999e-06 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.951805+0000", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.027798142 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.951921+0000", 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011533 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.952012+0000", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1704000000000002e-05 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958243+0000", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062309540000000004 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961089+0000", 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0028453240000000002 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.216 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.050946+0000", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.089856757999999995 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.050999+0000", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3619000000000003e-05 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051042+0000", 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2718e-05 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.217 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171317 14.a 14:52e1d2a5:::benchmark_data_smithi042_319156_object47360:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920320+0000", 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.22974548100001, 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1308062539999999, 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.218 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171317 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920320+0000", 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920320+0000", 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.219 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920320+0000", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8100000000000003e-07 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920322+0000", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9099999999999999e-06 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920329+0000", 2024-09-13T15:10:06.220 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5250000000000002e-06 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920334+0000", 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8300000000000003e-06 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924019+0000", 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003685308 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.221 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924022+0000", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4319999999999999e-06 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.954882+0000", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0308597020000001 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.954979+0000", 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7015000000000005e-05 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.222 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.955075+0000", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5618000000000005e-05 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958281+0000", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032065570000000001 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.966915+0000", 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086343459999999993 2024-09-13T15:10:06.223 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051049+0000", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.084133555999999998 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051093+0000", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3572999999999998e-05 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051126+0000", 2024-09-13T15:10:06.224 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3401000000000001e-05 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171319 14.a 14:582d2b3e:::benchmark_data_smithi042_319156_object47362:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920340+0000", 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.22972511, 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1308738140000001, 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.225 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171319 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920340+0000", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920340+0000", 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.226 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920341+0000", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6800000000000001e-07 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920342+0000", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8619999999999999e-06 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920348+0000", 2024-09-13T15:10:06.227 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0680000000000002e-06 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920354+0000", 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3890000000000004e-06 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924034+0000", 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036799469999999998 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.228 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924037+0000", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5030000000000002e-06 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958379+0000", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0343416139999999 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958484+0000", 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105202 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.229 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958588+0000", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000103471 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961135+0000", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025477249999999998 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.970032+0000", 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088966059999999996 2024-09-13T15:10:06.230 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051131+0000", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.081099560000000001 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051176+0000", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4230000000000002e-05 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051214+0000", 2024-09-13T15:10:06.231 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8269000000000002e-05 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171320 14.a 14:57cb108a:::benchmark_data_smithi042_319156_object47363:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920360+0000", 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.22970527499999, 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1309329830000001, 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.232 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171320 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920360+0000", 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.233 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920360+0000", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920360+0000", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.39e-07 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920362+0000", 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.764e-06 2024-09-13T15:10:06.234 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920368+0000", 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8429999999999998e-06 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920373+0000", 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0159999999999999e-06 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.235 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924049+0000", 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036757859999999999 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924052+0000", 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6090000000000001e-06 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961169+0000", 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.037116164 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.236 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961300+0000", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000131202 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961418+0000", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117926 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.963827+0000", 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024097789999999999 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.237 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.970118+0000", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062904989999999997 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051220+0000", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.081102010000000002 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051261+0000", 2024-09-13T15:10:06.238 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0862000000000003e-05 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051293+0000", 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2184000000000002e-05 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171321 14.a 14:51af6fc7:::benchmark_data_smithi042_319156_object47364:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.920379+0000", 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 133.229686663, 2024-09-13T15:10:06.239 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.130997104, 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171321 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.240 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920379+0000", 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920379+0000", 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920379+0000", 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7300000000000002e-07 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.241 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920381+0000", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.815e-06 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920387+0000", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1310000000000003e-06 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920393+0000", 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4740000000000002e-06 2024-09-13T15:10:06.242 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924074+0000", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036807580000000001 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924081+0000", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2450000000000001e-06 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.963886+0000", 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0398043530000001 2024-09-13T15:10:06.243 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.964025+0000", 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013934400000000001 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.964161+0000", 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013636 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.244 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.966949+0000", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027877510000000002 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.971817+0000", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048680820000000001 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051299+0000", 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.079481259999999998 2024-09-13T15:10:06.245 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051343+0000", 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4685000000000003e-05 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.051376+0000", 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2472999999999999e-05 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:224951 14.a 14:55c81995:::benchmark_data_smithi042_319156_object100994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e445)", 2024-09-13T15:10:06.246 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:08:55.134641+0000", 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 71.015423772999995, 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.083408021999999998, 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 224951 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.247 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134641+0000", 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134641+0000", 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134643+0000", 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4869999999999999e-06 2024-09-13T15:10:06.248 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134648+0000", 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4219999999999999e-06 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134662+0000", 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3151e-05 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.249 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.134669+0000", 2024-09-13T15:10:06.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:05 smithi159 ceph-mon[102519]: pgmap v1639: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 31 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T15:10:06.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:05 smithi159 ceph-mon[102519]: osdmap e470: 8 total, 8 up, 8 in 2024-09-13T15:10:06.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:10:06.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:05 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5179999999999998e-06 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.136612+0000", 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019431089999999999 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.136685+0000", 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2750000000000007e-05 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.251 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.136749+0000", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3997000000000004e-05 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.139270+0000", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025213000000000002 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.145883+0000", 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0066126190000000001 2024-09-13T15:10:06.252 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.217908+0000", 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.072024785999999993 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.217998+0000", 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0013999999999996e-05 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.253 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:55.218050+0000", 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1869000000000001e-05 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:231574 14.a 14:5d85d139:::benchmark_data_smithi042_319156_object107617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e445)", 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:09:00.426356+0000", 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 65.723709006000007, 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.083487046999999995, 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:06.254 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 231574 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426356+0000", 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.255 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426356+0000", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426357+0000", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7000000000000003e-07 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426362+0000", 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3719999999999998e-06 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.256 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426374+0000", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2106000000000001e-05 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.426381+0000", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5830000000000001e-06 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.430045+0000", 2024-09-13T15:10:06.257 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036635029999999998 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.430114+0000", 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.97e-05 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.430219+0000", 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000104799 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.258 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.434019+0000", 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0037999990000000001 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.440627+0000", 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0066081600000000001 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:06.259 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.509756+0000", 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.069128092000000002 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.509806+0000", 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0220999999999999e-05 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:09:00.509843+0000", 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.7542000000000001e-05 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.260 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.261 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.261 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:06.261 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:06.261 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:07.130 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:06 smithi042 ceph-mon[113926]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T15:10:07.131 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:06 smithi042 ceph-mon[108570]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T15:10:07.132 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-13T15:10:07.145 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:07.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:06 smithi159 ceph-mon[102519]: osdmap e471: 8 total, 8 up, 8 in 2024-09-13T15:10:08.028 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:07 smithi042 ceph-mon[108570]: pgmap v1642: 83 pgs: 3 remapped+peering, 80 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 96 B/s, 0 objects/s recovering 2024-09-13T15:10:08.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:07 smithi042 ceph-mon[108570]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T15:10:08.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:07 smithi042 ceph-mon[113926]: pgmap v1642: 83 pgs: 3 remapped+peering, 80 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 96 B/s, 0 objects/s recovering 2024-09-13T15:10:08.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:07 smithi042 ceph-mon[113926]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T15:10:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:07 smithi159 ceph-mon[102519]: pgmap v1642: 83 pgs: 3 remapped+peering, 80 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 96 B/s, 0 objects/s recovering 2024-09-13T15:10:08.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:07 smithi159 ceph-mon[102519]: osdmap e472: 8 total, 8 up, 8 in 2024-09-13T15:10:09.058 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:08 smithi042 ceph-mon[108570]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T15:10:09.059 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:08 smithi042 ceph-mon[113926]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stdout:mon.a: {} 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stdout:mon.b: {} 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stdout:mon.c: {} 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:10:09.173 INFO:teuthology.orchestra.run.smithi042.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-13T15:10:09.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:08 smithi159 ceph-mon[102519]: osdmap e473: 8 total, 8 up, 8 in 2024-09-13T15:10:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:09] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.51.0" 2024-09-13T15:10:09.921 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-13T15:10:10.236 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:09 smithi042 ceph-mon[113926]: pgmap v1645: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 150 B/s, 2 objects/s recovering 2024-09-13T15:10:10.236 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/418926894' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:10:10.236 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:09 smithi042 ceph-mon[108570]: pgmap v1645: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 150 B/s, 2 objects/s recovering 2024-09-13T15:10:10.236 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/418926894' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:10:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:09 smithi159 ceph-mon[102519]: pgmap v1645: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 150 B/s, 2 objects/s recovering 2024-09-13T15:10:10.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/418926894' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-13T15:10:10.773 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:12.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:11 smithi159 ceph-mon[102519]: pgmap v1646: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 101 B/s, 1 objects/s recovering 2024-09-13T15:10:12.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:11 smithi042 ceph-mon[108570]: pgmap v1646: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 101 B/s, 1 objects/s recovering 2024-09-13T15:10:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:11 smithi042 ceph-mon[113926]: pgmap v1646: 83 pgs: 1 peering, 3 remapped+peering, 79 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 101 B/s, 1 objects/s recovering 2024-09-13T15:10:12.481 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:12.481 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:10:12.481 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:10:12.481 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:13.518 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-13T15:10:14.054 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:14.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:13 smithi042 ceph-mon[108570]: pgmap v1647: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2024-09-13T15:10:14.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:13 smithi042 ceph-mon[113926]: pgmap v1647: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2024-09-13T15:10:14.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:13 smithi159 ceph-mon[102519]: pgmap v1647: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 B/s, 1 objects/s recovering 2024-09-13T15:10:14.885 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.1 to 1 (10000) 2024-09-13T15:10:15.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:15.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[108570]: pgmap v1648: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[108570]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[113926]: pgmap v1648: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:16.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:15 smithi042 ceph-mon[113926]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T15:10:16.157 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-13T15:10:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:15 smithi159 ceph-mon[102519]: pgmap v1648: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-09-13T15:10:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:15 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1076866315' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:16.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:15 smithi159 ceph-mon[102519]: osdmap e474: 8 total, 8 up, 8 in 2024-09-13T15:10:16.850 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:16.850 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:10:16.850 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:10:16.850 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:10:16.851 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:17.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:16 smithi042 ceph-mon[108570]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T15:10:17.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:16 smithi042 ceph-mon[113926]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T15:10:17.230 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:17.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:16 smithi159 ceph-mon[102519]: osdmap e475: 8 total, 8 up, 8 in 2024-09-13T15:10:17.712 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-13T15:10:17.943 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:17 smithi042 ceph-mon[113926]: pgmap v1651: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:17.944 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:17 smithi042 ceph-mon[108570]: pgmap v1651: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:18.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:17 smithi159 ceph-mon[102519]: pgmap v1651: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:18.489 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:20.052 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:19 smithi042 ceph-mon[108570]: pgmap v1652: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:20.053 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:19 smithi042 ceph-mon[113926]: pgmap v1652: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:20.053 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:19] "GET /metrics HTTP/1.1" 200 34146 "" "Prometheus/2.51.0" 2024-09-13T15:10:20.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:19 smithi159 ceph-mon[102519]: pgmap v1652: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:20.911 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-13T15:10:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:20 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:21.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:20 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:20 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:21.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:20 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:20 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-13T15:10:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:20 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:21.803 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[113926]: pgmap v1653: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[113926]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[108570]: pgmap v1653: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:22.046 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:21 smithi042 ceph-mon[108570]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T15:10:22.061 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:22.061 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:10:22.061 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:10:22.061 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129899 14.b 14:da4c7396:::benchmark_data_smithi042_319156_object5942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.71606816799999, 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6173837149999999, 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.062 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129899 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327270+0000", 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.063 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327271+0000", 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.468e-06 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327277+0000", 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2279999999999998e-06 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.064 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.328798+0000", 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015211020000000001 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.328813+0000", 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4973999999999999e-05 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.065 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331493+0000", 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026804400000000001 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331612+0000", 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000118748 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.066 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.331726+0000", 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011390199999999999 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351419+0000", 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.019692949000000001 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351430+0000", 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1068e-05 2024-09-13T15:10:22.067 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330782+0000", 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979351839 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330794+0000", 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2201000000000001e-05 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.068 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.899711+0000", 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5689170690000001 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for missing object", 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.899840+0000", 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012895199999999999 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.069 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.944553+0000", 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.044713389999999999 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.944654+0000", 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100385 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.071 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129914 14.b 14:d5295360:::benchmark_data_smithi042_319156_object5957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.70317777, 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6051032840000001, 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.072 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129914 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340160+0000", 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.073 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340162+0000", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9609999999999999e-06 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340170+0000", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5519999999999999e-06 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340182+0000", 2024-09-13T15:10:22.074 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2037e-05 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340191+0000", 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8230000000000001e-06 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340293+0000", 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000102279 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.075 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340441+0000", 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014848899999999999 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.340563+0000", 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122058 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351455+0000", 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010891675999999999 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.076 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351462+0000", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8750000000000002e-06 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330879+0000", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97941681000000003 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330890+0000", 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.164e-05 2024-09-13T15:10:22.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.900150+0000", 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5692593480000001 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for missing object", 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.900282+0000", 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132087 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.078 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.945195+0000", 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.044912967999999998 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.945264+0000", 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8681000000000002e-05 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129942 14.b 14:dcadd6a8:::benchmark_data_smithi042_319156_object5985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:10:22.079 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.65676695900001, 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5459006039999998, 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129942 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.080 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386572+0000", 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2699999999999999e-07 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.081 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386576+0000", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.211e-06 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386587+0000", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0947999999999999e-05 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386594+0000", 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2590000000000003e-06 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.082 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386667+0000", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3158999999999995e-05 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386677+0000", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4499999999999993e-06 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330921+0000", 2024-09-13T15:10:22.083 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94424369399999997 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.330929+0000", 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4279999999999996e-06 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.900317+0000", 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.569388164 2024-09-13T15:10:22.084 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.900472+0000", 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015476699999999999 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6,7", 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.901105+0000", 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00063332199999999996 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.085 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.915322+0000", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.014216352999999999 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.924425+0000", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0091029470000000001 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.924491+0000", 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6799999999999997e-05 2024-09-13T15:10:22.086 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.932298+0000", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078064559999999998 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.932352+0000", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4370000000000003e-05 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.932472+0000", 2024-09-13T15:10:22.087 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000119549 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129950 14.b 14:da02ad4e:::benchmark_data_smithi042_319156_object5993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 199.620757371, 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5223106189999998, 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.088 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129950 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422581+0000", 2024-09-13T15:10:22.089 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422582+0000", 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.491e-06 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422588+0000", 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0390000000000001e-06 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.090 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422602+0000", 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4138000000000001e-05 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422610+0000", 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3200000000000002e-06 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422668+0000", 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7917999999999997e-05 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.091 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.422676+0000", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9699999999999999e-06 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331166+0000", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90849047500000002 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331175+0000", 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8559999999999996e-06 2024-09-13T15:10:22.092 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.911277+0000", 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5801016670000001 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.911394+0000", 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117008 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6,7", 2024-09-13T15:10:22.093 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.911528+0000", 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013415000000000001 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.915539+0000", 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0040113730000000004 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.924435+0000", 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088957189999999998 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.094 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.924620+0000", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018522999999999999 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.944691+0000", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.020070745000000001 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.944783+0000", 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1830999999999995e-05 2024-09-13T15:10:22.095 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.944891+0000", 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000108689 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:170950 14.6 14:6eefb66f:::benchmark_data_smithi042_319156_object46993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.468281+0000", 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.57505686100001, 2024-09-13T15:10:22.096 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.22412643800000001, 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 170950 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468281+0000", 2024-09-13T15:10:22.097 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468281+0000", 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468282+0000", 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.02e-06 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.098 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468287+0000", 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9459999999999997e-06 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468301+0000", 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3623e-05 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.468308+0000", 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8220000000000001e-06 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.099 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.472167+0000", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038589050000000001 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for rw locks", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.472293+0000", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012640399999999999 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.637407+0000", 2024-09-13T15:10:22.100 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16511446199999999 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.637514+0000", 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010639199999999999 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.637694+0000", 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017991799999999999 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.101 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.646674+0000", 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0089802030000000008 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.650607+0000", 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039335780000000001 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.688446+0000", 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.037838352999999998 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.102 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.688519+0000", 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2699000000000003e-05 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.692408+0000", 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003889113 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171090 14.6 14:685aef0a:::benchmark_data_smithi042_319156_object47133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:22.103 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.651962+0000", 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.391375834, 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.17369373900000001, 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171090 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.104 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.651962+0000", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.651962+0000", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.651964+0000", 2024-09-13T15:10:22.105 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6279999999999999e-06 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.651969+0000", 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0209999999999999e-06 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.652013+0000", 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3649999999999997e-05 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.106 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.652020+0000", 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6650000000000003e-06 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.668728+0000", 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.016707593 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for rw locks", 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.668833+0000", 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000105228 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.107 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.780507+0000", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.11167425 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.780618+0000", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011061 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.780763+0000", 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014540899999999999 2024-09-13T15:10:22.108 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.791415+0000", 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.010651598 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.791476+0000", 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1153000000000002e-05 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.109 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.821771+0000", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.030295228 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.821850+0000", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8243000000000003e-05 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.825656+0000", 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038064629999999999 2024-09-13T15:10:22.110 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171307 14.b 14:df57d176:::benchmark_data_smithi042_319156_object47350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.919292+0000", 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.12404632600001, 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.050138966, 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.111 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171307 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919292+0000", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919292+0000", 2024-09-13T15:10:22.112 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919293+0000", 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6899999999999999e-06 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919300+0000", 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.229e-06 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.113 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919322+0000", 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1956000000000001e-05 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.919334+0000", 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1898e-05 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.920001+0000", 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00066760400000000003 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.114 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.920014+0000", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3198e-05 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.919887+0000", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99987221599999998 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.919897+0000", 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0314e-05 2024-09-13T15:10:22.115 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.947690+0000", 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0277934179999999 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.947826+0000", 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013559800000000001 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.116 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.947963+0000", 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013739500000000001 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.953464+0000", 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0055008490000000004 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961408+0000", 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079439980000000007 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.117 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969304+0000", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078958139999999993 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969384+0000", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9808000000000006e-05 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969431+0000", 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6981000000000003e-05 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.118 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171329 14.b 14:d651fb84:::benchmark_data_smithi042_319156_object47372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.973214+0000", 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.07012421499999, 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.996331337, 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171329 2024-09-13T15:10:22.119 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973214+0000", 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973214+0000", 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.120 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.121 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973215+0000", 2024-09-13T15:10:22.121 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2550000000000001e-06 2024-09-13T15:10:22.121 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.121 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.122 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.122 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973220+0000", 2024-09-13T15:10:22.122 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8459999999999999e-06 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973232+0000", 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1943999999999999e-05 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973239+0000", 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7710000000000001e-06 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.123 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973303+0000", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4645000000000002e-05 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.973312+0000", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.003e-06 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.919924+0000", 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94661178000000001 2024-09-13T15:10:22.124 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.919930+0000", 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0599999999999996e-06 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.953502+0000", 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.033571544 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.125 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.953590+0000", 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7978000000000004e-05 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.953655+0000", 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5026999999999999e-05 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961518+0000", 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0078631029999999998 2024-09-13T15:10:22.126 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961527+0000", 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.302e-06 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969474+0000", 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079468279999999995 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.127 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969514+0000", 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0138000000000001e-05 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969545+0000", 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1112999999999998e-05 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.128 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171341 14.b 14:dbbd7afb:::benchmark_data_smithi042_319156_object47384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.010714+0000", 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.03262451099999, 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9589379820000001, 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171341 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.129 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010714+0000", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010714+0000", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010715+0000", 2024-09-13T15:10:22.130 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2589999999999999e-06 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010720+0000", 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1259999999999997e-06 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010733+0000", 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3284e-05 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.131 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010740+0000", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.221e-06 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010793+0000", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2809000000000002e-05 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.010799+0000", 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5849999999999999e-06 2024-09-13T15:10:22.132 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920777+0000", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90997827899999995 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.920791+0000", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3531e-05 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.957541+0000", 2024-09-13T15:10:22.133 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.036750354 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.957655+0000", 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000114447 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.957788+0000", 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013304199999999999 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.134 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961556+0000", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003767163 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969249+0000", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076936649999999997 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969582+0000", 2024-09-13T15:10:22.135 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00033244699999999998 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969621+0000", 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8951e-05 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969652+0000", 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0818999999999997e-05 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.136 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171343 14.b 14:d1f45363:::benchmark_data_smithi042_319156_object47386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.014611+0000", 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.02872665199999, 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.963191712, 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.137 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171343 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014611+0000", 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014611+0000", 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.138 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014613+0000", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2869999999999999e-06 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014617+0000", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5650000000000003e-06 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014631+0000", 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3862999999999999e-05 2024-09-13T15:10:22.139 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014639+0000", 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3930000000000002e-06 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014726+0000", 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.7108999999999996e-05 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.140 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.014735+0000", 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7820000000000005e-06 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.921765+0000", 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.90703 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.921773+0000", 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0450000000000001e-06 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.141 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961587+0000", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0398135909999999 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961672+0000", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4987000000000005e-05 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.961764+0000", 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1564999999999995e-05 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.142 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969684+0000", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.007920669 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969719+0000", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4545999999999998e-05 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977706+0000", 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079874290000000008 2024-09-13T15:10:22.143 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977757+0000", 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0871999999999997e-05 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977803+0000", 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6008999999999999e-05 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.144 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171344 14.b 14:d1966e83:::benchmark_data_smithi042_319156_object47387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.016492+0000", 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 150.026846326, 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9614399790000001, 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171344 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.145 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016492+0000", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016492+0000", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016492+0000", 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3399999999999998e-07 2024-09-13T15:10:22.146 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016496+0000", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6679999999999999e-06 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016506+0000", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0359e-05 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016513+0000", 2024-09-13T15:10:22.147 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2369999999999997e-06 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016554+0000", 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1335000000000003e-05 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.016559+0000", 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8940000000000001e-06 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.148 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.921799+0000", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.905239672 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.921806+0000", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0729999999999999e-06 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.965359+0000", 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.043552853 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.149 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.965450+0000", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1055000000000001e-05 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.965548+0000", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.8398000000000002e-05 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969752+0000", 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0042042099999999999 2024-09-13T15:10:22.150 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977811+0000", 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0080584799999999998 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977853+0000", 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1857000000000003e-05 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.151 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977899+0000", 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6641000000000003e-05 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977932+0000", 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2512999999999999e-05 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171348 14.b 14:d5b18d1f:::benchmark_data_smithi042_319156_object47391:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2024-09-13T15:10:22.152 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:52.982116+0000", 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 149.06122233100001, 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0011072059999999, 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171348 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.153 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982116+0000", 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982116+0000", 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982117+0000", 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.517e-06 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.154 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982123+0000", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.241e-06 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982136+0000", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2767e-05 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982142+0000", 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1659999999999999e-06 2024-09-13T15:10:22.155 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982204+0000", 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1452999999999996e-05 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.982210+0000", 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5549999999999999e-06 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.156 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969783+0000", 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98757207499999999 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.969903+0000", 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012000799999999999 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,6", 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.970010+0000", 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107639 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.157 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977940+0000", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0079299009999999996 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.977979+0000", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9385999999999999e-05 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.983136+0000", 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0051562090000000001 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.158 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.983170+0000", 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4538e-05 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.983223+0000", 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2750999999999997e-05 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.159 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171810 14.6 14:62fad142:::benchmark_data_smithi042_319156_object47853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.519267+0000", 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.524071467, 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18762034599999999, 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171810 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.160 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519267+0000", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519267+0000", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519268+0000", 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0890000000000001e-06 2024-09-13T15:10:22.161 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519273+0000", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8509999999999998e-06 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519284+0000", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1786e-05 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519292+0000", 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3629999999999996e-06 2024-09-13T15:10:22.162 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519362+0000", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9931999999999994e-05 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519525+0000", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000163373 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519726+0000", 2024-09-13T15:10:22.163 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000200953 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.526614+0000", 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0068880069999999998 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.526668+0000", 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4345999999999998e-05 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.164 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.702612+0000", 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.175943302 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.702689+0000", 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7458000000000003e-05 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.706887+0000", 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041978859999999996 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.165 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171815 14.6 14:6614daa0:::benchmark_data_smithi042_319156_object47858:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.519712+0000", 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.52362575800001, 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19441461900000001, 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.166 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171815 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519712+0000", 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519712+0000", 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.167 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519713+0000", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7000000000000004e-07 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519717+0000", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2300000000000002e-06 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519734+0000", 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6993000000000001e-05 2024-09-13T15:10:22.168 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.519744+0000", 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3440000000000007e-06 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.523343+0000", 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035997730000000001 2024-09-13T15:10:22.169 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.523483+0000", 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013980500000000001 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.523640+0000", 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015699900000000001 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.170 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.526729+0000", 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030885449999999998 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.526889+0000", 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016035499999999999 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.706945+0000", 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18005589 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.171 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.707156+0000", 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00021091799999999999 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.714127+0000", 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069709969999999996 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.172 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171835 14.6 14:6c9872b5:::benchmark_data_smithi042_319156_object47878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.173 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.545325+0000", 2024-09-13T15:10:22.173 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.49801275499999, 2024-09-13T15:10:22.173 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.172680786, 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.176 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171835 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545325+0000", 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545325+0000", 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.177 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545327+0000", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5540000000000001e-06 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545332+0000", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5439999999999998e-06 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545350+0000", 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7110999999999999e-05 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.178 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545357+0000", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4619999999999999e-06 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545717+0000", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035973100000000002 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.545853+0000", 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013583599999999999 2024-09-13T15:10:22.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.546001+0000", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014823899999999999 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.551313+0000", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0053126170000000004 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.551328+0000", 2024-09-13T15:10:22.180 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4622e-05 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.714235+0000", 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16290685799999999 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.714298+0000", 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3362999999999996e-05 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.181 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.718006+0000", 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003707849 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171845 14.6 14:60390f9f:::benchmark_data_smithi042_319156_object47888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.554833+0000", 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.48850529200001, 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16691974600000001, 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.182 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171845 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554833+0000", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.183 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554833+0000", 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554834+0000", 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.477e-06 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554839+0000", 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5839999999999996e-06 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.184 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554850+0000", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1418e-05 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.554865+0000", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5138e-05 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.560020+0000", 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0051551080000000003 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.185 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.560164+0000", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014348400000000001 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.560304+0000", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013993499999999999 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.569408+0000", 2024-09-13T15:10:22.186 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0091044730000000001 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.573586+0000", 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041779299999999998 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.718078+0000", 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.14449211000000001 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.187 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.718141+0000", 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2564000000000006e-05 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.721753+0000", 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003611525 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171855 14.6 14:66e5703f:::benchmark_data_smithi042_319156_object47898:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.564607+0000", 2024-09-13T15:10:22.188 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.47873098299999, 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16098660200000001, 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171855 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564607+0000", 2024-09-13T15:10:22.189 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564607+0000", 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564608+0000", 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3599999999999999e-06 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.190 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564614+0000", 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6579999999999999e-06 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564626+0000", 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2286000000000001e-05 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564635+0000", 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2770000000000005e-06 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.191 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564696+0000", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1235999999999994e-05 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564829+0000", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000132755 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564978+0000", 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014918599999999999 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.192 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.573625+0000", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086473980000000006 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.574479+0000", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00085345400000000002 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.721868+0000", 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.14738959800000001 2024-09-13T15:10:22.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.721930+0000", 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1488000000000005e-05 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.725594+0000", 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036639060000000002 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.194 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171857 14.6 14:6d515d5f:::benchmark_data_smithi042_319156_object47900:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e424)", 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:54.564671+0000", 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 147.47866714400001, 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.1665826, 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171857 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.195 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564671+0000", 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564671+0000", 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.196 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564671+0000", 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0399999999999996e-07 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564674+0000", 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8260000000000001e-06 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564682+0000", 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6420000000000007e-06 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.197 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.564688+0000", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6690000000000003e-06 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.569445+0000", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047574139999999997 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.569629+0000", 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00018357399999999999 2024-09-13T15:10:22.198 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.569804+0000", 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017516 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.573696+0000", 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038921939999999999 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.199 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.574578+0000", 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00088195500000000002 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.725669+0000", 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.15109075899999999 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.725735+0000", 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5977999999999997e-05 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.200 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:54.731254+0000", 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.005518925 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:196225 14.6 14:6ac552da:::benchmark_data_smithi042_319156_object72268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:08:24.154467+0000", 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 117.88887155099999, 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18333901599999999, 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.201 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 196225 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154467+0000", 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.202 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154467+0000", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154468+0000", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.285e-06 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154473+0000", 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4450000000000004e-06 2024-09-13T15:10:22.203 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154487+0000", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3946e-05 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.154494+0000", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2450000000000001e-06 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.169819+0000", 2024-09-13T15:10:22.204 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015324449 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for rw locks", 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.169916+0000", 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7449999999999997e-05 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.290789+0000", 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.12087221099999999 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.205 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.290908+0000", 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011969 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.291049+0000", 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000140777 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.303819+0000", 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012770045000000001 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.206 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.303839+0000", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0423000000000001e-05 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.334319+0000", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.030479701000000001 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.334381+0000", 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2086000000000002e-05 2024-09-13T15:10:22.207 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:24.337806+0000", 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0034242629999999999 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:201830 14.6 14:6cd2b451:::benchmark_data_smithi042_319156_object77873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e433)", 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:08:30.323087+0000", 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 111.720251262, 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.16061979400000001, 2024-09-13T15:10:22.208 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 201830 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323087+0000", 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.209 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323087+0000", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323088+0000", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5300000000000003e-07 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323092+0000", 2024-09-13T15:10:22.210 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6269999999999999e-06 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323104+0000", 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1652e-05 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.323111+0000", 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0509999999999999e-06 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.211 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.324750+0000", 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001639176 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for rw locks", 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.324899+0000", 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014843699999999999 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.419677+0000", 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.094778012999999994 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.212 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.419773+0000", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6249999999999995e-05 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,6", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.419930+0000", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000157515 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.427356+0000", 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0074254339999999999 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.213 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.430428+0000", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030723880000000001 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.480888+0000", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.050460118999999998 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.480951+0000", 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2774999999999994e-05 2024-09-13T15:10:22.214 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:08:30.483707+0000", 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0027555040000000002 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:22.215 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:21 smithi159 ceph-mon[102519]: pgmap v1653: 83 pgs: 83 active+clean; 582 KiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:21 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/819466174' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-13T15:10:22.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:21 smithi159 ceph-mon[102519]: osdmap e476: 8 total, 8 up, 8 in 2024-09-13T15:10:23.266 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-13T15:10:23.290 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:24.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:23 smithi042 ceph-mon[108570]: pgmap v1655: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:24.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:23 smithi042 ceph-mon[113926]: pgmap v1655: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:24.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:23 smithi159 ceph-mon[102519]: pgmap v1655: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:24.496 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.005 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:24 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-09-13T15:10:25.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:24 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]} : dispatch 2024-09-13T15:10:25.930 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.2 to 1 (10000) 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: pgmap v1656: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1855856540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: pgmap v1656: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1855856540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:25 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: pgmap v1656: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-13T15:10:26.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-09-13T15:10:26.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-09-13T15:10:26.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 2]}]': finished 2024-09-13T15:10:26.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: osdmap e477: 8 total, 8 up, 8 in 2024-09-13T15:10:26.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1855856540' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:25 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:26.781 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:26.781 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:10:26.781 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:10:26.781 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:27.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:26 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:27.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:26 smithi159 ceph-mon[102519]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T15:10:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:26 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:27.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:26 smithi042 ceph-mon[108570]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T15:10:27.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:26 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:27.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:26 smithi042 ceph-mon[113926]: osdmap e478: 8 total, 8 up, 8 in 2024-09-13T15:10:27.559 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-13T15:10:27.987 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-13T15:10:28.043 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:28.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:27 smithi042 ceph-mon[108570]: pgmap v1659: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:28.098 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:27 smithi042 ceph-mon[113926]: pgmap v1659: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:28.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:27 smithi159 ceph-mon[102519]: pgmap v1659: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:28.807 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:29.826 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:29.826 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:10:29.826 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:10:29.826 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:10:29.826 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:29.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:29] "GET /metrics HTTP/1.1" 200 34121 "" "Prometheus/2.51.0" 2024-09-13T15:10:30.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:29 smithi159 ceph-mon[102519]: pgmap v1660: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:30.297 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:29 smithi042 ceph-mon[113926]: pgmap v1660: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:30.298 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:29 smithi042 ceph-mon[108570]: pgmap v1660: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:30.757 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-13T15:10:31.637 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:32.033 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:31 smithi042 ceph-mon[108570]: pgmap v1661: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:32.033 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:31 smithi042 ceph-mon[113926]: pgmap v1661: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:32.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:31 smithi159 ceph-mon[102519]: pgmap v1661: 83 pgs: 1 activating, 82 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:32.943 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-13T15:10:33.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:32 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:33.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:32 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:33.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:32 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-13T15:10:34.214 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[113926]: pgmap v1662: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:34.215 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:34.215 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[113926]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T15:10:34.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[108570]: pgmap v1662: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:34.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:34.215 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:33 smithi042 ceph-mon[108570]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T15:10:34.240 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-13T15:10:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:33 smithi159 ceph-mon[102519]: pgmap v1662: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:33 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1068125942' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-13T15:10:34.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:33 smithi159 ceph-mon[102519]: osdmap e479: 8 total, 8 up, 8 in 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.457 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60162 13.2 13:418d4a79:::benchmark_data_smithi042_286876_object60161:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 541.58851616799996, 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0028493329999999, 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60162 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.458 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865485+0000", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865487+0000", 2024-09-13T15:10:34.459 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8250000000000001e-06 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865508+0000", 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0947000000000001e-05 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865524+0000", 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6052000000000001e-05 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.460 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865532+0000", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5989999999999996e-06 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865587+0000", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5349999999999997e-05 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.865596+0000", 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9719999999999998e-06 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.461 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.859408+0000", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99381218500000001 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860985+0000", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001576446 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.861455+0000", 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00047016700000000001 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.462 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864756+0000", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003301114 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865824+0000", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010678669999999999 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868257+0000", 2024-09-13T15:10:34.463 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024326009999999999 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868292+0000", 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.5324000000000003e-05 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.868335+0000", 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2883999999999998e-05 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.464 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60179 13.b 13:da237103:::benchmark_data_smithi042_286876_object60178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 541.58281116900002, 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.004732119, 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:10:34.465 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60179 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871190+0000", 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.466 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871192+0000", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.708e-06 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871215+0000", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2867e-05 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871233+0000", 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.7872000000000001e-05 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.467 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871244+0000", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1304000000000001e-05 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871328+0000", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3912999999999994e-05 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871348+0000", 2024-09-13T15:10:34.468 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9386e-05 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863786+0000", 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99243854099999995 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.863945+0000", 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015913099999999999 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:10:34.469 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.864079+0000", 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013408800000000001 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872431+0000", 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0083515080000000005 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872656+0000", 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000225345 2024-09-13T15:10:34.470 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875824+0000", 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0031679709999999999 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875879+0000", 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4401000000000003e-05 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.471 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875923+0000", 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4084e-05 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60186 13.b 13:db94921c:::benchmark_data_smithi042_286876_object60185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 541.57930325500001, 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.001358341, 2024-09-13T15:10:34.472 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60186 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.473 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874698+0000", 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874700+0000", 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.782e-06 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.474 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874727+0000", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.6758e-05 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874746+0000", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8569000000000001e-05 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.874754+0000", 2024-09-13T15:10:34.475 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.1769999999999998e-06 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875021+0000", 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026734699999999997 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.875031+0000", 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0098999999999999e-05 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.476 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866528+0000", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.991496618 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866700+0000", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017193800000000001 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.866831+0000", 2024-09-13T15:10:34.477 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013130999999999999 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.872614+0000", 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0057832559999999996 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875962+0000", 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003347629 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.478 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.875974+0000", 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2177999999999999e-05 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876022+0000", 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8270999999999999e-05 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.876057+0000", 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.4409000000000002e-05 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.479 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129900 14.1 14:82ff686f:::benchmark_data_smithi042_319156_object5943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.327306+0000", 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.126695324, 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4170785270000001, 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129900 2024-09-13T15:10:34.480 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327306+0000", 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327306+0000", 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.481 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327309+0000", 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.199e-06 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327315+0000", 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2589999999999997e-06 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327333+0000", 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8320000000000001e-05 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.482 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327343+0000", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.7510000000000003e-06 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327411+0000", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8518999999999999e-05 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327537+0000", 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012516200000000001 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.483 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.327837+0000", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000300136 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.332580+0000", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0047432400000000001 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.332934+0000", 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00035432699999999998 2024-09-13T15:10:34.484 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351805+0000", 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.018870990000000001 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351816+0000", 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0389e-05 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.485 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328393+0000", 2024-09-13T15:10:34.486 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97657689299999995 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328404+0000", 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1454000000000001e-05 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.744297+0000", 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4158926949999999 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.487 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.744385+0000", 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8193000000000001e-05 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129907 14.1 14:85d05e5d:::benchmark_data_smithi042_319156_object5950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.338691+0000", 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.11531043299999, 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4319542570000001, 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.488 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129907 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338691+0000", 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.489 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338691+0000", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338693+0000", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.415e-06 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338696+0000", 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.726e-06 2024-09-13T15:10:34.490 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338741+0000", 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4724000000000001e-05 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338749+0000", 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5299999999999999e-06 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.491 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351838+0000", 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013089783000000001 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351845+0000", 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1380000000000004e-06 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.492 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328431+0000", 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97658609200000002 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328439+0000", 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0690000000000002e-06 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.744685+0000", 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4162458359999999 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.493 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.744795+0000", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011053900000000001 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2,6", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.745317+0000", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00052149200000000003 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760624+0000", 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.015307433 2024-09-13T15:10:34.494 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760638+0000", 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3858e-05 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760672+0000", 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.3744000000000001e-05 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.495 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770440+0000", 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097685110000000006 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770507+0000", 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7139999999999998e-05 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770645+0000", 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013822699999999999 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.496 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129912 14.1 14:8e6fb5a5:::benchmark_data_smithi042_319156_object5955:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.338760+0000", 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.11524200299999, 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4322270769999998, 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129912 2024-09-13T15:10:34.497 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338760+0000", 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338760+0000", 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.498 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338760+0000", 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5000000000000003e-07 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338763+0000", 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2600000000000001e-06 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338771+0000", 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2699999999999999e-06 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.499 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.338776+0000", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3650000000000003e-06 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351885+0000", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.013109054 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.351892+0000", 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5409999999999997e-06 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.500 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328462+0000", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.97657061300000003 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328470+0000", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8499999999999994e-06 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.751548+0000", 2024-09-13T15:10:34.501 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.423077919 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.751676+0000", 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012782099999999999 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2,6", 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.751891+0000", 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000215319 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.502 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760712+0000", 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0088206919999999998 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770334+0000", 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0096223179999999995 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770387+0000", 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2469000000000002e-05 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.503 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770785+0000", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039857599999999999 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770852+0000", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6235999999999994e-05 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770987+0000", 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013522400000000001 2024-09-13T15:10:34.504 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129916 14.1 14:8eeaaf5d:::benchmark_data_smithi042_319156_object5959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.356226+0000", 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.09777571699999, 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4149949479999999, 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.505 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129916 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356226+0000", 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356226+0000", 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.506 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356227+0000", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.314e-06 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356230+0000", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.2329999999999998e-06 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356239+0000", 2024-09-13T15:10:34.507 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4370000000000007e-06 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.356247+0000", 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4360000000000002e-06 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.358715+0000", 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024676139999999999 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.508 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.358724+0000", 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.4469999999999995e-06 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328494+0000", 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.969769145 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328501+0000", 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.5499999999999997e-06 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.509 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760746+0000", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4322447110000001 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.760873+0000", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012753600000000001 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2,6", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.761135+0000", 2024-09-13T15:10:34.510 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026143299999999999 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770401+0000", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0092659279999999997 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770691+0000", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00028985300000000002 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.770736+0000", 2024-09-13T15:10:34.511 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5228999999999997e-05 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771036+0000", 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00030009500000000001 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771099+0000", 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3094000000000003e-05 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.512 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771221+0000", 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012189300000000001 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129928 14.1 14:85107a1b:::benchmark_data_smithi042_319156_object5971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.370281+0000", 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.08372077300001, 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4092421210000001, 2024-09-13T15:10:34.513 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129928 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370281+0000", 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.514 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370281+0000", 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370283+0000", 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6869999999999999e-06 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.515 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370287+0000", 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3930000000000001e-06 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370300+0000", 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2629000000000001e-05 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370308+0000", 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.541e-06 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.516 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370396+0000", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.8196999999999996e-05 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.370410+0000", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3482000000000001e-05 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328524+0000", 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.95811444300000004 2024-09-13T15:10:34.517 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328532+0000", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3000000000000004e-06 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.767139+0000", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4386074879999999 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.767269+0000", 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000129608 2024-09-13T15:10:34.518 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2,6", 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.767391+0000", 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000122485 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771268+0000", 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038768959999999999 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.519 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771284+0000", 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5733e-05 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.777926+0000", 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0066421880000000003 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779345+0000", 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0014195360000000001 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.520 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779407+0000", 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1978999999999997e-05 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779523+0000", 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000115536 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.521 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129929 14.f 14:fada1f88:::benchmark_data_smithi042_319156_object5972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.371002+0000", 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.08299924299999, 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.074525215, 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129929 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.522 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371002+0000", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371002+0000", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371003+0000", 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7299999999999995e-07 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.523 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371007+0000", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.6270000000000002e-06 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371015+0000", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.2239999999999995e-06 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371020+0000", 2024-09-13T15:10:34.524 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6569999999999996e-06 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371079+0000", 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9460999999999999e-05 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371088+0000", 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9560000000000003e-06 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.525 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331731+0000", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.96064333400000002 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.331741+0000", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5929999999999995e-06 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.435620+0000", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.103879072 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.526 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.435723+0000", 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010331399999999999 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.435989+0000", 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026536600000000001 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.441632+0000", 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0056428549999999996 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.527 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445211+0000", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035795470000000002 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445267+0000", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5488999999999997e-05 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445332+0000", 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5065000000000001e-05 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.528 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445401+0000", 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9035000000000006e-05 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445528+0000", 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000126947 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.529 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129930 14.a 14:57e294a1:::benchmark_data_smithi042_319156_object5973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.371352+0000", 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.08264920400001, 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1145089810000002, 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129930 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.530 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371352+0000", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371352+0000", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371353+0000", 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4199999999999995e-07 2024-09-13T15:10:34.531 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371357+0000", 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.0459999999999999e-06 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371385+0000", 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8025e-05 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.532 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371398+0000", 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3055e-05 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371443+0000", 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4872e-05 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.371451+0000", 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7039999999999994e-06 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.533 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328299+0000", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.95684820900000001 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328311+0000", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1895e-05 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.466018+0000", 2024-09-13T15:10:34.534 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1377075969999999 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.468636+0000", 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0026178529999999998 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,4,5", 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.468921+0000", 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00028471399999999998 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.535 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.481886+0000", 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012964654000000001 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.483991+0000", 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021050230000000001 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.484003+0000", 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2294e-05 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.536 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.485728+0000", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001725397 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.485790+0000", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1909999999999995e-05 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.485861+0000", 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1091000000000001e-05 2024-09-13T15:10:34.537 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129933 14.1 14:8d621833:::benchmark_data_smithi042_319156_object5976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.375840+0000", 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.07816145999999, 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4038791270000002, 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.538 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129933 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375840+0000", 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375840+0000", 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.539 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375841+0000", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2079999999999999e-06 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375847+0000", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.288e-06 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375870+0000", 2024-09-13T15:10:34.540 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3193999999999998e-05 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375878+0000", 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7770000000000002e-06 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375937+0000", 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9599999999999999e-05 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.541 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.375947+0000", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6800000000000005e-06 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328563+0000", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.95261586200000004 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328573+0000", 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0118e-05 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.542 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771328+0000", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.442755295 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771481+0000", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000152855 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2,6", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.771634+0000", 2024-09-13T15:10:34.543 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015340699999999999 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.777939+0000", 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0063048169999999999 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.777978+0000", 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.8727999999999999e-05 2024-09-13T15:10:34.544 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.778760+0000", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00078243599999999996 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779583+0000", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00082284700000000003 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779641+0000", 2024-09-13T15:10:34.545 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7565000000000002e-05 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.779719+0000", 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8449999999999996e-05 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129941 14.f 14:f71f92f8:::benchmark_data_smithi042_319156_object5984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.386569+0000", 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.06743238600001, 2024-09-13T15:10:34.546 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0609883240000002, 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129941 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386569+0000", 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.547 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386569+0000", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386571+0000", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.252e-06 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386575+0000", 2024-09-13T15:10:34.548 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9459999999999997e-06 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386590+0000", 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4450999999999999e-05 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386597+0000", 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.165e-06 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.549 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386668+0000", 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1365000000000005e-05 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.386678+0000", 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3479999999999993e-06 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.333758+0000", 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94708020100000001 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.550 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.333767+0000", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0340000000000002e-06 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.441685+0000", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1079181199999999 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.441776+0000", 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.1000000000000003e-05 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.551 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.441850+0000", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4232000000000003e-05 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445545+0000", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036951140000000002 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445577+0000", 2024-09-13T15:10:34.552 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1458999999999998e-05 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445678+0000", 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101277 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447451+0000", 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.001772745 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.553 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447495+0000", 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4396e-05 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447558+0000", 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2218999999999995e-05 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129943 14.f 14:fb1d3f3b:::benchmark_data_smithi042_319156_object5986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.391670+0000", 2024-09-13T15:10:34.554 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.06233128, 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.056003032, 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129943 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391670+0000", 2024-09-13T15:10:34.555 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391670+0000", 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391672+0000", 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.127e-06 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.556 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391676+0000", 2024-09-13T15:10:34.557 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5240000000000002e-06 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391688+0000", 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1582e-05 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391696+0000", 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9840000000000001e-06 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.558 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391774+0000", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.8597000000000007e-05 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.391785+0000", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0597e-05 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.333786+0000", 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.94200078899999995 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.559 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.333791+0000", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5979999999999996e-06 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.443492+0000", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1097013069999999 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.443588+0000", 2024-09-13T15:10:34.560 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.5692000000000005e-05 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.443665+0000", 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6396999999999994e-05 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445592+0000", 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019270489999999999 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.561 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445752+0000", 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016014600000000001 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.445870+0000", 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117804 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447588+0000", 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0017187249999999999 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.562 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447617+0000", 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8750000000000001e-05 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.447673+0000", 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6363999999999998e-05 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.563 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129947 14.a 14:503fec17:::benchmark_data_smithi042_319156_object5990:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.415129+0000", 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.03887238900001, 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0720280199999999, 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129947 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.564 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415129+0000", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415129+0000", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415131+0000", 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3820000000000001e-06 2024-09-13T15:10:34.565 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415136+0000", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9579999999999998e-06 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415147+0000", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1239e-05 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415154+0000", 2024-09-13T15:10:34.566 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3189999999999997e-06 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415212+0000", 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8223999999999997e-05 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415219+0000", 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8870000000000003e-06 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.567 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328337+0000", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.91311736200000004 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328346+0000", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3530000000000001e-06 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.481924+0000", 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.153578126 2024-09-13T15:10:34.568 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.482050+0000", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012616 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,4,5", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.482195+0000", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014515700000000001 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.485869+0000", 2024-09-13T15:10:34.569 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0036736690000000001 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.486512+0000", 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00064265500000000001 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.486770+0000", 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00025802900000000001 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.570 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487040+0000", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00026996000000000002 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487086+0000", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5816999999999999e-05 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487157+0000", 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1723000000000004e-05 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.571 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:129948 14.a 14:58c3c38a:::benchmark_data_smithi042_319156_object5991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:02.415396+0000", 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 212.03860600900001, 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0719067280000001, 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.572 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 129948 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415396+0000", 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415396+0000", 2024-09-13T15:10:34.573 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415396+0000", 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.8700000000000005e-07 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415400+0000", 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.252e-06 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415412+0000", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2692e-05 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415417+0000", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1580000000000004e-06 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415464+0000", 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6621e-05 2024-09-13T15:10:34.575 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:02.415477+0000", 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2473999999999999e-05 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328378+0000", 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.91290184900000004 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.576 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:03.328719+0000", 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034109899999999998 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.484026+0000", 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1553061069999999 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.484091+0000", 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.5831e-05 2024-09-13T15:10:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,4,5", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.484162+0000", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0885999999999999e-05 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.485874+0000", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0017118560000000001 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.486558+0000", 2024-09-13T15:10:34.578 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00068418099999999996 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.486901+0000", 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034271900000000001 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487184+0000", 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000282763 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.579 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487238+0000", 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3810000000000001e-05 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:04.487302+0000", 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4743e-05 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.580 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:139793 14.3 14:c17a4e34:::benchmark_data_smithi042_319156_object15836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:15.485228+0000", 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 198.968773932, 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.004412361, 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 139793 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.581 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485228+0000", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485228+0000", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485230+0000", 2024-09-13T15:10:34.582 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8250000000000001e-06 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485235+0000", 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.0810000000000001e-06 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485247+0000", 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.24e-05 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.583 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485255+0000", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7589999999999997e-06 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485372+0000", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000117662 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:15.485380+0000", 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.6690000000000006e-06 2024-09-13T15:10:34.584 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.484195+0000", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.998814804 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.484329+0000", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013383899999999999 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 4,5", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.484467+0000", 2024-09-13T15:10:34.585 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013856 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.487412+0000", 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029445489999999999 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.487458+0000", 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6363000000000003e-05 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.586 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.489582+0000", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021237999999999999 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.489611+0000", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8634999999999999e-05 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:16.489640+0000", 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9414999999999999e-05 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.587 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171298 14.a 14:555b3e19:::benchmark_data_smithi042_319156_object47341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.894709+0000", 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 162.559292661, 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.029530874, 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.588 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171298 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894709+0000", 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894709+0000", 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.589 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894710+0000", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2200000000000004e-07 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894712+0000", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.543e-06 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894719+0000", 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1670000000000002e-06 2024-09-13T15:10:34.590 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.894727+0000", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7680000000000008e-06 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.900962+0000", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062347089999999997 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.901086+0000", 2024-09-13T15:10:34.591 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012403499999999999 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,4,5", 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.901216+0000", 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000130343 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.913998+0000", 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.012782152 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.592 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.920274+0000", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0062762449999999997 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.929798+0000", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0095238839999999998 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.929811+0000", 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.235e-05 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.593 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924224+0000", 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99441310100000002 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924240+0000", 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6055000000000001e-05 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171305 14.f 14:f18e4aa3:::benchmark_data_smithi042_319156_object47348:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:34.594 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.913996+0000", 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 162.540006108, 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0599878540000001, 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171305 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.595 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.913996+0000", 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.913996+0000", 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.913996+0000", 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.34e-07 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.596 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.913998+0000", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.5189999999999999e-06 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.914006+0000", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2629999999999997e-06 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.914011+0000", 2024-09-13T15:10:34.597 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7370000000000003e-06 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.914097+0000", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.5451999999999994e-05 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.914229+0000", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013224499999999999 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,5,7", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.914367+0000", 2024-09-13T15:10:34.598 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013761200000000001 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.922584+0000", 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0082169010000000004 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.932301+0000", 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0097170060000000003 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.599 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.932312+0000", 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1551e-05 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.970437+0000", 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.038124555 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for missing object", 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.970538+0000", 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000101275 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.600 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.973930+0000", 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003391682 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.973983+0000", 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3622000000000003e-05 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.601 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171311 14.1 14:881fd174:::benchmark_data_smithi042_319156_object47354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.936085+0000", 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 162.517916786, 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.02262824, 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171311 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.602 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936085+0000", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936085+0000", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936086+0000", 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4559999999999999e-06 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.603 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936091+0000", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5900000000000001e-06 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936363+0000", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00027237000000000002 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936374+0000", 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0986e-05 2024-09-13T15:10:34.604 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936444+0000", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9752000000000006e-05 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.936460+0000", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5568000000000001e-05 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.928599+0000", 2024-09-13T15:10:34.605 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99213916599999996 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.928995+0000", 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00039579099999999999 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.951371+0000", 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.022376167 2024-09-13T15:10:34.606 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.951513+0000", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000142688 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 1,2", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.951637+0000", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012351199999999999 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958252+0000", 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0066154730000000002 2024-09-13T15:10:34.607 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958268+0000", 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5172e-05 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958615+0000", 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00034767899999999999 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:10:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958672+0000", 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6418000000000001e-05 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:53.958713+0000", 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.1452000000000002e-05 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.36129.0:171312 14.a 14.a745c7ba (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:07:51.937102+0000", 2024-09-13T15:10:34.609 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 162.51690002800001, 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98717491199999996, 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.36129", 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 171312 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:10:34.610 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937102+0000", 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937102+0000", 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937103+0000", 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.632e-06 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.611 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937109+0000", 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9220000000000002e-06 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937125+0000", 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5724999999999999e-05 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937140+0000", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4715999999999999e-05 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937220+0000", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.0037000000000006e-05 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:51.937230+0000", 2024-09-13T15:10:34.613 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0797e-05 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924269+0000", 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98703812800000001 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:10:34.614 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:07:52.924277+0000", 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9549999999999992e-06 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:10:34.615 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:10:34.968 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:35.505 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-13T15:10:35.723 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:36.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:35 smithi042 ceph-mon[113926]: pgmap v1664: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:36.025 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:35 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:36.025 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:35 smithi042 ceph-mon[108570]: pgmap v1664: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:36.025 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:35 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:36.145 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:35 smithi159 ceph-mon[102519]: pgmap v1664: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:36.145 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:35 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:37.511 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:37.511 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:37.511 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T15:10:37.511 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:37.955 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.3 to 1 (10000) 2024-09-13T15:10:38.148 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:37 smithi159 ceph-mon[102519]: pgmap v1665: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:38.148 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:37 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3432507883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.148 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:37 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.170 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-13T15:10:38.253 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[113926]: pgmap v1665: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:38.254 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3432507883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.254 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[108570]: pgmap v1665: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:38.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3432507883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.254 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:37 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:38.372 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:38.731 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-13T15:10:38.942 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:38 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:38.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:38 smithi159 ceph-mon[102519]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T15:10:39.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:38 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:39.278 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:38 smithi042 ceph-mon[113926]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T15:10:39.278 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:38 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:39.278 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:38 smithi042 ceph-mon[108570]: osdmap e480: 8 total, 8 up, 8 in 2024-09-13T15:10:39.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:39] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-09-13T15:10:39.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:39 smithi159 ceph-mon[102519]: pgmap v1667: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:40.016 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:40.017 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:40.017 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:10:40.017 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:10:40.017 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:40.320 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:39 smithi042 ceph-mon[113926]: pgmap v1667: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:40.321 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:39 smithi042 ceph-mon[108570]: pgmap v1667: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:40.629 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-13T15:10:40.831 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:41.965 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-13T15:10:42.239 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:41 smithi159 ceph-mon[102519]: pgmap v1668: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:42.239 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:41 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:42.281 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:41 smithi042 ceph-mon[113926]: pgmap v1668: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:42.281 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:41 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:42.282 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:41 smithi042 ceph-mon[108570]: pgmap v1668: 83 pgs: 83 active+clean; 582 KiB data, 9.4 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:42.282 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:41 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-13T15:10:42.457 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:42.457 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:10:42.457 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60169 13.e 13:71178aff:::benchmark_data_smithi042_286876_object60168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 549.59592942899997, 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0134129039999999, 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.458 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60169 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858285+0000", 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.459 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858283+0000", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858317+0000", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3337999999999998e-05 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858325+0000", 2024-09-13T15:10:42.460 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5189999999999992e-06 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858331+0000", 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7440000000000004e-06 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.461 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858652+0000", 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000321031 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.858664+0000", 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1865000000000001e-05 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.462 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.859035+0000", 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.000370776 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.860409+0000", 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0013744390000000001 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.860580+0000", 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00017043399999999999 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.463 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867875+0000", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0072958679999999996 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867909+0000", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3773999999999999e-05 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871610+0000", 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037006669999999999 2024-09-13T15:10:42.464 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871656+0000", 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6369000000000003e-05 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871698+0000", 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1471999999999999e-05 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.465 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.466 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60199 13.e 13:7465222c:::benchmark_data_smithi042_286876_object60198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 549.56941665500005, 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98705518800000003, 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.468 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60199 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.469 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884798+0000", 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884797+0000", 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.470 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884856+0000", 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9181999999999997e-05 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884867+0000", 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0471e-05 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884873+0000", 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7689999999999999e-06 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.471 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884932+0000", 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8471000000000003e-05 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.884944+0000", 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2184999999999999e-05 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.472 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.862746+0000", 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97780187600000001 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.862928+0000", 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000182528 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.863148+0000", 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000219435 2024-09-13T15:10:42.473 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.867943+0000", 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047948289999999996 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871732+0000", 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0037891190000000001 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.474 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871779+0000", 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7583000000000002e-05 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.475 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871821+0000", 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1870999999999999e-05 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871853+0000", 2024-09-13T15:10:42.476 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1609000000000002e-05 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:60203 13.e 13:7b415ff7:::benchmark_data_smithi042_286876_object60202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 549.56712760999994, 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98575690599999999, 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.477 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 60203 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887087+0000", 2024-09-13T15:10:42.478 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887084+0000", 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999971 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887151+0000", 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7595000000000006e-05 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.479 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887187+0000", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5825999999999999e-05 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887195+0000", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4139999999999997e-06 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887225+0000", 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0437e-05 2024-09-13T15:10:42.480 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:32.887232+0000", 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.815e-06 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865319+0000", 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97808694299999999 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.481 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865466+0000", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014752600000000001 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 5,7", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.865621+0000", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015456000000000001 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.871737+0000", 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061158360000000004 2024-09-13T15:10:42.482 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872278+0000", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000541349 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872756+0000", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00047819699999999998 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872806+0000", 2024-09-13T15:10:42.483 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9539999999999997e-05 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:01:33.872844+0000", 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7774000000000001e-05 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213354 13.5 13:a90db7da:::benchmark_data_smithi042_286876_object89853:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.484 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.44157969100002, 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.057659903, 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213354 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.485 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012635+0000", 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012634+0000", 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.486 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012636+0000", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2019999999999998e-06 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012642+0000", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2340000000000002e-06 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.012646+0000", 2024-09-13T15:10:42.487 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1659999999999996e-06 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024265+0000", 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.011619378 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024273+0000", 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9999999999999996e-06 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.488 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059337+0000", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.035064153 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059442+0000", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000104627 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.059586+0000", 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014403699999999999 2024-09-13T15:10:42.489 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063004+0000", 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003418291 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066853+0000", 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003848803 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.490 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070209+0000", 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003355575 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070256+0000", 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7079000000000001e-05 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070295+0000", 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8770000000000003e-05 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.491 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213357 13.5 13:a5fa48f4:::benchmark_data_smithi042_286876_object89856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.43848029499998, 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.06210454, 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.492 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213357 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015734+0000", 2024-09-13T15:10:42.493 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015733+0000", 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015741+0000", 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5000000000000002e-06 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.494 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015749+0000", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7280000000000001e-06 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.015756+0000", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5690000000000001e-06 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024305+0000", 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0085489969999999992 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.495 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.024312+0000", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6760000000000001e-06 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063040+0000", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0387282419999999 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063151+0000", 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000111705 2024-09-13T15:10:42.496 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.063734+0000", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00058296499999999996 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070400+0000", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0066657829999999998 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070456+0000", 2024-09-13T15:10:42.497 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5537e-05 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077760+0000", 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073039439999999997 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077809+0000", 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8980000000000002e-05 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.498 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077839+0000", 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9844000000000001e-05 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213369 13.5 13:af8813b1:::benchmark_data_smithi042_286876_object89868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.41936520600001, 2024-09-13T15:10:42.499 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.043141705, 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213369 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.500 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034849+0000", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034851+0000", 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0219999999999999e-06 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.501 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034856+0000", 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.887e-06 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.034860+0000", 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4989999999999996e-06 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.502 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037222+0000", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023623400000000001 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037232+0000", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.3619999999999995e-06 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066884+0000", 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0296525050000001 2024-09-13T15:10:42.503 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.504 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.504 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.504 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.066958+0000", 2024-09-13T15:10:42.504 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3930999999999994e-05 2024-09-13T15:10:42.504 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.067044+0000", 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5995999999999995e-05 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070444+0000", 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034000940000000002 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.505 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070463+0000", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.821e-05 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077940+0000", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0074770849999999996 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077970+0000", 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9966000000000001e-05 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.506 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077991+0000", 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1169e-05 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213373 13.5 13:aceb1f16:::benchmark_data_smithi042_286876_object89872:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.41741013400002, 2024-09-13T15:10:42.507 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0412968069999999, 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213373 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.508 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036804+0000", 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.509 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036807+0000", 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5159999999999999e-06 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036814+0000", 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2679999999999997e-06 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.036820+0000", 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5829999999999997e-06 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.510 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037254+0000", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00043378200000000002 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.037259+0000", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.8089999999999998e-06 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070497+0000", 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0332373290000001 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.511 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070622+0000", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000124762 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.070772+0000", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015073899999999999 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077900+0000", 2024-09-13T15:10:42.512 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0071275740000000002 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.077998+0000", 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8414999999999999e-05 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078059+0000", 2024-09-13T15:10:42.513 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0547999999999997e-05 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078083+0000", 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3706e-05 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078101+0000", 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.844e-05 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.514 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213380 13.5 13:a8878734:::benchmark_data_smithi042_286876_object89879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.408572736, 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0325521609999999, 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213380 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.515 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045642+0000", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045640+0000", 2024-09-13T15:10:42.516 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045646+0000", 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5490000000000003e-06 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045659+0000", 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3157e-05 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.517 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045666+0000", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5740000000000001e-06 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045708+0000", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2543999999999998e-05 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.045717+0000", 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6859999999999995e-06 2024-09-13T15:10:42.518 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074126+0000", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0284085199999999 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074235+0000", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000109467 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.074355+0000", 2024-09-13T15:10:42.519 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000120294 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078004+0000", 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036484629999999998 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078032+0000", 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8059e-05 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.520 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078135+0000", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000102682 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078162+0000", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.743e-05 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078194+0000", 2024-09-13T15:10:42.521 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1788000000000001e-05 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213388 13.5 13:aaa5fe06:::benchmark_data_smithi042_286876_object89887:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.39445837300002, 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0326702839999999, 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.522 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213388 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059756+0000", 2024-09-13T15:10:42.523 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059755+0000", 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059760+0000", 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9969999999999998e-06 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.524 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059767+0000", 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2150000000000004e-06 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059772+0000", 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4450000000000004e-06 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.525 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059822+0000", 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9412000000000001e-05 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.059831+0000", 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5489999999999995e-06 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078221+0000", 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.018389196 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.526 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078296+0000", 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4994999999999996e-05 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.078408+0000", 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000112861 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.527 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.085959+0000", 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0075507389999999999 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.085995+0000", 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5725000000000001e-05 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092381+0000", 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0063859390000000002 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.528 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092405+0000", 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3830999999999999e-05 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092426+0000", 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1597e-05 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.529 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213390 13.5 13:a04508b5:::benchmark_data_smithi042_286876_object89889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.39412176000002, 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0323922619999999, 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213390 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.530 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060093+0000", 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.531 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060092+0000", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060096+0000", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3640000000000001e-06 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060105+0000", 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7280000000000001e-06 2024-09-13T15:10:42.532 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060110+0000", 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1250000000000001e-06 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060142+0000", 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.2400999999999997e-05 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.533 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.060150+0000", 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2980000000000003e-06 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.082957+0000", 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.022807571 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.083061+0000", 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000103991 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.534 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.083139+0000", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.729e-05 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086032+0000", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0028935480000000001 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092305+0000", 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0062728180000000003 2024-09-13T15:10:42.535 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092433+0000", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012801699999999999 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092456+0000", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2623e-05 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092485+0000", 2024-09-13T15:10:42.536 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9295999999999999e-05 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213423 13.5 13:afde02ce:::benchmark_data_smithi042_286876_object89922:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.35918747300002, 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99753285599999997, 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.537 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213423 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095027+0000", 2024-09-13T15:10:42.538 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095025+0000", 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095031+0000", 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2669999999999997e-06 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.539 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095040+0000", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1819999999999995e-06 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.095046+0000", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9209999999999997e-06 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.096636+0000", 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0015904809999999999 2024-09-13T15:10:42.540 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.096644+0000", 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9289999999999998e-06 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086059+0000", 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98941457200000005 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.541 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086136+0000", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7562e-05 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.086222+0000", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5393e-05 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092250+0000", 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0060284639999999999 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.542 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092374+0000", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012345800000000001 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092493+0000", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000119329 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092526+0000", 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3481e-05 2024-09-13T15:10:42.543 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092560+0000", 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3201000000000003e-05 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213445 13.5 13:aa84eeca:::benchmark_data_smithi042_286876_object89944:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.33836532700002, 2024-09-13T15:10:42.544 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.98705722900000004, 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213445 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.545 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115849+0000", 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115848+0000", 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.546 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115852+0000", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3340000000000003e-06 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115862+0000", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6600000000000007e-06 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.115869+0000", 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4300000000000003e-06 2024-09-13T15:10:42.547 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.116666+0000", 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00079784300000000001 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.116674+0000", 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8399999999999995e-06 2024-09-13T15:10:42.548 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089371+0000", 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97269705500000003 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089445+0000", 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3355000000000005e-05 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.549 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.089530+0000", 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5198999999999995e-05 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092344+0000", 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002814263 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.550 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.096027+0000", 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0036830320000000001 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102852+0000", 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0068251869999999999 2024-09-13T15:10:42.551 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102884+0000", 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1445000000000001e-05 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102906+0000", 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2470999999999999e-05 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.552 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213452 13.5 13:af44a94b:::benchmark_data_smithi042_286876_object89951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.32992939500002, 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97874794499999995, 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213452 2024-09-13T15:10:42.553 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124285+0000", 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.554 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124284+0000", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124288+0000", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5460000000000002e-06 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124299+0000", 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0478e-05 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.555 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124304+0000", 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7899999999999999e-06 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124339+0000", 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5309999999999999e-05 2024-09-13T15:10:42.556 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.124349+0000", 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9590000000000007e-06 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092591+0000", 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.96824188899999997 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.557 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092671+0000", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9610000000000005e-05 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.092793+0000", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012206600000000001 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.096058+0000", 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032655029999999999 2024-09-13T15:10:42.558 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102845+0000", 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067864350000000004 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102979+0000", 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013441700000000001 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.559 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.103007+0000", 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8161999999999999e-05 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.103033+0000", 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5748000000000001e-05 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.45788.0:213538 13.5 13:a51335ba:::benchmark_data_smithi042_286876_object90037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e380)", 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:10:42.560 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 389.21401998499999, 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.88139930200000005, 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.45788", 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 213538 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.561 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240194+0000", 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.562 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240198+0000", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.566e-06 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240212+0000", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4149e-05 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240219+0000", 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3999999999999997e-06 2024-09-13T15:10:42.563 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240262+0000", 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3509e-05 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:13.240271+0000", 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7960000000000001e-06 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:42.564 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099494+0000", 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.85922312099999998 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099633+0000", 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013862799999999999 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.099753+0000", 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012005100000000001 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.565 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.102972+0000", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032194789999999999 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.117182+0000", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.014209816 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121539+0000", 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043566949999999998 2024-09-13T15:10:42.566 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121568+0000", 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9218e-05 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:14.121594+0000", 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.565e-05 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:42.567 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:42.568 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:42.706 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-13T15:10:42.921 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:43.187 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:42 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:43.187 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:42 smithi159 ceph-mon[102519]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T15:10:43.209 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-13T15:10:43.223 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:42 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:43.223 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:42 smithi042 ceph-mon[113926]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T15:10:43.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:42 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/15927995' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-13T15:10:43.223 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:42 smithi042 ceph-mon[108570]: osdmap e481: 8 total, 8 up, 8 in 2024-09-13T15:10:43.410 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:44.100 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:43 smithi042 ceph-mon[108570]: pgmap v1670: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:44.100 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:43 smithi042 ceph-mon[113926]: pgmap v1670: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:44.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:43 smithi159 ceph-mon[102519]: pgmap v1670: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:44.976 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.4 to 1 (10000) 2024-09-13T15:10:45.115 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:45.115 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:45.115 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T15:10:45.115 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:44 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2909450726' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:44 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:44 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2909450726' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:44 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:44 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2909450726' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:44 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:45.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-13T15:10:45.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-13T15:10:45.872 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:45.964 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:45 smithi159 ceph-mon[102519]: pgmap v1671: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:45 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:45.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:45 smithi159 ceph-mon[102519]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T15:10:46.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[108570]: pgmap v1671: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:46.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:46.306 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[108570]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T15:10:46.306 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[113926]: pgmap v1671: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:46.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:46.307 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:45 smithi042 ceph-mon[113926]: osdmap e482: 8 total, 8 up, 8 in 2024-09-13T15:10:47.645 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:47.645 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:47.645 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:10:47.645 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:10:47.645 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:47.982 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:47 smithi042 ceph-mon[108570]: pgmap v1673: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:48.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:47 smithi159 ceph-mon[102519]: pgmap v1673: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:48.298 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-13T15:10:48.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:47 smithi042 ceph-mon[113926]: pgmap v1673: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:48.498 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:48.989 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-13T15:10:49.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:48 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:49.319 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:48 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:49.319 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:48 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-13T15:10:49.714 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-13T15:10:49.929 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:49.989 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:49] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-09-13T15:10:50.186 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:50.186 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:10:50.186 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:10:50.186 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:194472 14.8 e433/407 14:1e0e8953:::benchmark_data_smithi042_319156_object70515:head v 433'12061, mlcod=433'12059)", 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:22.113243+0000", 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 148.07033323100001, 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037081078000000003, 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.187 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113243+0000", 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113243+0000", 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.188 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113242+0000", 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113253+0000", 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0893000000000001e-05 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113269+0000", 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5818999999999999e-05 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.189 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113276+0000", 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2180000000000002e-06 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113425+0000", 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000149667 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.190 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.113470+0000", 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4765999999999998e-05 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.150281+0000", 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.036811071000000001 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:22.150324+0000", 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2864999999999997e-05 2024-09-13T15:10:50.191 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:195734 14.3 e433/416 14:c1c33344:::benchmark_data_smithi042_319156_object71777:head v 433'12436, mlcod=433'12433)", 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:23.560549+0000", 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 146.62302715999999, 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.035021494, 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.192 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560549+0000", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560549+0000", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560548+0000", 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.193 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560560+0000", 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1766999999999999e-05 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560577+0000", 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7370000000000001e-05 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.194 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.560585+0000", 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4930000000000001e-06 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.562636+0000", 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002051443 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.195 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.562657+0000", 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.037e-05 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.196 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.595528+0000", 2024-09-13T15:10:50.197 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.032871816999999998 2024-09-13T15:10:50.197 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.197 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.197 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.197 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.595571+0000", 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2216999999999999e-05 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:195735 14.3 e433/416 14:c8bdc2e3:::benchmark_data_smithi042_319156_object71778:head v 433'12437, mlcod=433'12433)", 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:23.563494+0000", 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 146.620082266, 2024-09-13T15:10:50.198 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.032113933999999997, 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563494+0000", 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563494+0000", 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.199 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563493+0000", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563503+0000", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.781e-06 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563518+0000", 2024-09-13T15:10:50.200 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4878000000000001e-05 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.563523+0000", 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3689999999999998e-06 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.575824+0000", 2024-09-13T15:10:50.201 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012300510000000001 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.575845+0000", 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0605999999999999e-05 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.595575+0000", 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019730272 2024-09-13T15:10:50.202 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:23.595608+0000", 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.311e-05 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:202916 14.0 e433/395 14:076d068e:::benchmark_data_smithi042_319156_object78959:head v 433'12466, mlcod=433'12463)", 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:31.490690+0000", 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 138.692885969, 2024-09-13T15:10:50.203 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.032145607999999999, 2024-09-13T15:10:50.204 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490690+0000", 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490690+0000", 2024-09-13T15:10:50.205 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490690+0000", 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.206 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490696+0000", 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9540000000000001e-06 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490704+0000", 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9030000000000004e-06 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.207 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.490708+0000", 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7010000000000004e-06 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.493696+0000", 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0029880190000000002 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.208 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.493751+0000", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4623999999999997e-05 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.522703+0000", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028952043 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:31.522836+0000", 2024-09-13T15:10:50.209 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013292999999999999 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:204682 14.a e434/423 14:5190a52b:::benchmark_data_smithi042_319156_object80725:head v 434'13036, mlcod=434'13033)", 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:33.323585+0000", 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 136.85999149700001, 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.032383383000000002, 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.210 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323585+0000", 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323585+0000", 2024-09-13T15:10:50.211 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.212 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.212 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.212 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.212 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323584+0000", 2024-09-13T15:10:50.212 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323591+0000", 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5869999999999998e-06 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.213 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323608+0000", 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7059e-05 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323616+0000", 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8509999999999999e-06 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.214 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323782+0000", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00016653800000000001 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.323849+0000", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6787999999999998e-05 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.355940+0000", 2024-09-13T15:10:50.215 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.032090714999999999 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.355968+0000", 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8609e-05 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.216 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:204681 14.2 e434/431 14:4e685884:::benchmark_data_smithi042_319156_object80724:head v 434'12852, mlcod=434'12849)", 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:33.324651+0000", 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 136.85892525599999, 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.035429545999999999, 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324651+0000", 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.217 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324651+0000", 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324650+0000", 2024-09-13T15:10:50.218 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324657+0000", 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5220000000000004e-06 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324670+0000", 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3512999999999999e-05 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.219 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.324677+0000", 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1029999999999998e-06 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.331668+0000", 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0069916040000000002 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.220 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.331719+0000", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0775000000000001e-05 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.359993+0000", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028273979000000001 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:33.360081+0000", 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7707999999999994e-05 2024-09-13T15:10:50.221 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:226403 14.d e445/395 14:b7244cb8:::benchmark_data_smithi042_319156_object102446:head v 445'14264, mlcod=445'14261)", 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:56.313248+0000", 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 113.870328761, 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.020481948, 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.222 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313248+0000", 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313248+0000", 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.223 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313247+0000", 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313254+0000", 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8059999999999997e-06 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313269+0000", 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5339e-05 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.224 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.313276+0000", 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3119999999999996e-06 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.316869+0000", 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035925990000000001 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.225 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.316913+0000", 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4715000000000001e-05 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.333698+0000", 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.016784871999999999 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.226 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.333729+0000", 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1131999999999999e-05 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:226757 14.d e445/395 14:b0a8e52a:::benchmark_data_smithi042_319156_object102800:head v 445'14288, mlcod=445'14285)", 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:56.565499+0000", 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 113.618077528, 2024-09-13T15:10:50.227 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019873575000000001, 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565499+0000", 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565499+0000", 2024-09-13T15:10:50.228 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565498+0000", 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.229 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565504+0000", 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6200000000000004e-06 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565532+0000", 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7727999999999999e-05 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.230 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.565538+0000", 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2689999999999996e-06 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.566933+0000", 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001394609 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.566995+0000", 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2262999999999997e-05 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.231 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.585293+0000", 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.018297798000000001 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:56.585372+0000", 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9705999999999999e-05 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.232 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:227672 14.d e445/395 14:bad56028:::benchmark_data_smithi042_319156_object103715:head v 445'14349, mlcod=445'14345)", 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:57.330689+0000", 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 112.852886923, 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.018805313000000001, 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.233 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330689+0000", 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330689+0000", 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.234 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330689+0000", 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330696+0000", 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5959999999999998e-06 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330711+0000", 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5027e-05 2024-09-13T15:10:50.235 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.330718+0000", 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.2219999999999996e-06 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.345429+0000", 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.014710378 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.236 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.345474+0000", 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5231000000000001e-05 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.349452+0000", 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0039776760000000003 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:57.349495+0000", 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2993e-05 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.237 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:228981 14.a e445/423 14:50e1fe20:::benchmark_data_smithi042_319156_object105024:head v 445'14593, mlcod=445'14589)", 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:58.487497+0000", 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 111.696079783, 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019145565, 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.238 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487497+0000", 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487497+0000", 2024-09-13T15:10:50.239 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487496+0000", 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.240 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487503+0000", 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9469999999999997e-06 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487519+0000", 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6741000000000001e-05 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.241 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487526+0000", 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6610000000000003e-06 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487568+0000", 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2222000000000003e-05 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.487586+0000", 2024-09-13T15:10:50.242 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.772e-05 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.506335+0000", 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.018748984999999999 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.506642+0000", 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00030719999999999999 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.243 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:229022 14.7 e445/436 14:e29fea28:::benchmark_data_smithi042_319156_object105065:head v 445'13671, mlcod=445'13668)", 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:58.521045+0000", 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 111.662531538, 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.021507422000000002, 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521045+0000", 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.244 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521045+0000", 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521045+0000", 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967296 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.245 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521049+0000", 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1230000000000002e-06 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521054+0000", 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0980000000000001e-06 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.246 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.521058+0000", 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.053e-06 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.523115+0000", 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.002057171 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.247 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.248 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.523133+0000", 2024-09-13T15:10:50.248 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7921e-05 2024-09-13T15:10:50.248 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.248 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.248 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.542532+0000", 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019398839000000001 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.542552+0000", 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0446999999999999e-05 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.249 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:229634 14.d e445/395 14:be341eff:::benchmark_data_smithi042_319156_object105677:head v 445'14466, mlcod=445'14464)", 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:58.965129+0000", 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 111.21844758899999, 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.023185124000000001, 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.250 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965129+0000", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965129+0000", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965128+0000", 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.251 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965135+0000", 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.849e-06 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965144+0000", 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5750000000000007e-06 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.252 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965151+0000", 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5300000000000002e-06 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965206+0000", 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4932000000000002e-05 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.253 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.965223+0000", 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7682000000000001e-05 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.988215+0000", 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022991919999999999 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.254 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:58.988314+0000", 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8536000000000007e-05 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.255 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:229868 14.a e445/423 14:59ef468a:::benchmark_data_smithi042_319156_object105911:head v 445'14646, mlcod=445'14642)", 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.141305+0000", 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 111.042271419, 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025322470999999999, 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.141305+0000", 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.256 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.141305+0000", 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.141304+0000", 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.257 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.141311+0000", 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7719999999999997e-06 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.142312+0000", 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0010008090000000001 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.258 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.259 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.142320+0000", 2024-09-13T15:10:50.259 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6559999999999998e-06 2024-09-13T15:10:50.259 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.260 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.142720+0000", 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00039926300000000002 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.142743+0000", 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3804000000000001e-05 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.261 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.166525+0000", 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.023781493000000001 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.166627+0000", 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000102539 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:229871 14.a e445/423 14:5abbd0a4:::benchmark_data_smithi042_319156_object105914:head v 445'14647, mlcod=445'14642)", 2024-09-13T15:10:50.262 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.146450+0000", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 111.037126149, 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022632558000000001, 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146450+0000", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146450+0000", 2024-09-13T15:10:50.263 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146449+0000", 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146457+0000", 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6159999999999996e-06 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.264 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146467+0000", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.9839999999999996e-06 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.146473+0000", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9259999999999997e-06 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.147108+0000", 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00063529000000000005 2024-09-13T15:10:50.265 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.147175+0000", 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7089999999999996e-05 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.169021+0000", 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.021845521 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.266 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.169083+0000", 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2162000000000006e-05 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:229942 14.a e445/423 14:5d99cb36:::benchmark_data_smithi042_319156_object105985:head v 445'14652, mlcod=445'14649)", 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.193057+0000", 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 110.99051928900001, 2024-09-13T15:10:50.267 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.021110065000000001, 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193057+0000", 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193057+0000", 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.268 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193056+0000", 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193063+0000", 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6409999999999996e-06 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.269 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193072+0000", 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0880000000000001e-06 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.193077+0000", 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2410000000000003e-06 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.270 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.197869+0000", 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047919290000000003 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.197919+0000", 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9808999999999997e-05 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.214097+0000", 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.016177438999999998 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.271 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.214167+0000", 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0569999999999997e-05 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:230429 14.a e445/423 14:5a8bb7cf:::benchmark_data_smithi042_319156_object106472:head v 445'14680, mlcod=445'14675)", 2024-09-13T15:10:50.272 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.563132+0000", 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 110.620444628, 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.021037759, 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563132+0000", 2024-09-13T15:10:50.273 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563132+0000", 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563131+0000", 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.274 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563138+0000", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1949999999999997e-06 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563149+0000", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0976e-05 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.563155+0000", 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3010000000000003e-06 2024-09-13T15:10:50.275 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.565084+0000", 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0019285940000000001 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.276 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.565100+0000", 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6186000000000001e-05 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.584138+0000", 2024-09-13T15:10:50.277 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019038204999999999 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.584169+0000", 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1143000000000003e-05 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:230443 14.a e445/423 14:5138bd6f:::benchmark_data_smithi042_319156_object106486:head v 445'14681, mlcod=445'14676)", 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.566557+0000", 2024-09-13T15:10:50.278 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 110.617019261, 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.028967698, 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566557+0000", 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566557+0000", 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.279 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566556+0000", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566563+0000", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0659999999999998e-06 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566577+0000", 2024-09-13T15:10:50.280 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.398e-05 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.566584+0000", 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.562e-06 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.281 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571175+0000", 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0045914750000000002 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571195+0000", 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9281999999999999e-05 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.595423+0000", 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.024228632999999999 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.282 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.595525+0000", 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010153999999999999 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:230454 14.a e445/423 14:571a9477:::benchmark_data_smithi042_319156_object106497:head v 445'14682, mlcod=445'14676)", 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:08:59.571920+0000", 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 110.611656719, 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.023659899000000002, 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.283 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571920+0000", 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571920+0000", 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.284 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571919+0000", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571926+0000", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4529999999999996e-06 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571941+0000", 2024-09-13T15:10:50.285 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4742e-05 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.571948+0000", 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.669e-06 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.575437+0000", 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0034893599999999999 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.286 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.576155+0000", 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00071748500000000002 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.595560+0000", 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019405242 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:08:59.595579+0000", 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9700000000000001e-05 2024-09-13T15:10:50.287 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:232260 14.d e445/395 14:bc5f2d36:::benchmark_data_smithi042_319156_object108303:head v 445'14630, mlcod=445'14624)", 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:09:01.046199+0000", 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 109.137376901, 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.023119627, 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.288 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046199+0000", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046199+0000", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046199+0000", 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:50.289 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046205+0000", 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3199999999999996e-06 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046214+0000", 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4190000000000008e-06 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.290 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.046220+0000", 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7830000000000004e-06 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.291 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.048969+0000", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027491 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.048992+0000", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2572999999999999e-05 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.069285+0000", 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.020293499 2024-09-13T15:10:50.292 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.069319+0000", 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3574000000000001e-05 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:232266 14.d e445/395 14:b775f754:::benchmark_data_smithi042_319156_object108309:head v 445'14631, mlcod=445'14624)", 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:09:01.049581+0000", 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 109.133995024, 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019759793000000001, 2024-09-13T15:10:50.293 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049581+0000", 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:50.294 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049581+0000", 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049581+0000", 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049586+0000", 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0239999999999997e-06 2024-09-13T15:10:50.295 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049600+0000", 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.472e-05 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.049606+0000", 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1659999999999999e-06 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.296 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.052054+0000", 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024477819999999999 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:50.297 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.052070+0000", 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6075000000000001e-05 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.069322+0000", 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.017251788000000001 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:09:01.069341+0000", 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8977000000000001e-05 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.298 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.299 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:50.299 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:50.299 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:50.299 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:49 smithi159 ceph-mon[102519]: pgmap v1674: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:50.299 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:49 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:50.299 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:49 smithi159 ceph-mon[102519]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T15:10:50.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[108570]: pgmap v1674: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:50.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:50.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[108570]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T15:10:50.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[113926]: pgmap v1674: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:50.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3353593142' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-13T15:10:50.340 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:49 smithi042 ceph-mon[113926]: osdmap e483: 8 total, 8 up, 8 in 2024-09-13T15:10:50.979 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-13T15:10:51.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:50 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:51.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:50 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:51.180 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:51.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:50 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:10:51.998 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.5 to 1 (10000) 2024-09-13T15:10:52.247 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:52 smithi159 ceph-mon[102519]: pgmap v1676: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:52.247 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:52 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3397713051' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.247 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:52 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[108570]: pgmap v1676: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3397713051' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[113926]: pgmap v1676: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3397713051' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:52 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:52.794 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-13T15:10:52.800 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:52.800 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:52.801 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T15:10:52.801 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:53.007 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:53.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:53 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:53.099 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:53 smithi042 ceph-mon[108570]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T15:10:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:53 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:53.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:53 smithi042 ceph-mon[113926]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T15:10:53.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:53 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:53.250 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:53 smithi159 ceph-mon[102519]: osdmap e484: 8 total, 8 up, 8 in 2024-09-13T15:10:53.427 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-13T15:10:53.636 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:54.108 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:54 smithi159 ceph-mon[102519]: pgmap v1678: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:54.160 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:54 smithi042 ceph-mon[108570]: pgmap v1678: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:54.161 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:54 smithi042 ceph-mon[113926]: pgmap v1678: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:55.011 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-13T15:10:55.156 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:55 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:55.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:55 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:55.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:55 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-13T15:10:55.428 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:55.429 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:10:55.429 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:10:55.429 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:10:55.429 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:55.751 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-13T15:10:56.018 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[108570]: pgmap v1679: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:56.057 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[108570]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[113926]: pgmap v1679: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[113926]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T15:10:56.302 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:56 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:56 smithi159 ceph-mon[102519]: pgmap v1679: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:56 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1162243687' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-13T15:10:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:56 smithi159 ceph-mon[102519]: osdmap e485: 8 total, 8 up, 8 in 2024-09-13T15:10:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:56 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:10:56.642 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:10:56.914 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:10:57.294 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:57 smithi159 ceph-mon[102519]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T15:10:57.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:57 smithi042 ceph-mon[113926]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T15:10:57.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:57 smithi042 ceph-mon[108570]: osdmap e486: 8 total, 8 up, 8 in 2024-09-13T15:10:58.039 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:58 smithi159 ceph-mon[102519]: pgmap v1682: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:58.198 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:58 smithi042 ceph-mon[113926]: pgmap v1682: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:58.199 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:58 smithi042 ceph-mon[108570]: pgmap v1682: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:10:58.851 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:10:58.851 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:10:58.851 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:10:58.851 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168456 14.2 14:42d955ca:::benchmark_data_smithi042_319156_object44499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.872504+0000", 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.86064835400001, 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0874103239999999, 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.852 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168456 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872504+0000", 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872504+0000", 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.853 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872503+0000", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872507+0000", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.41e-06 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872524+0000", 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7439999999999999e-05 2024-09-13T15:10:58.854 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.872530+0000", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7640000000000002e-06 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.877327+0000", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047969429999999997 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.877527+0000", 2024-09-13T15:10:58.855 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000200161 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3,5", 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.877680+0000", 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00015255800000000001 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.883317+0000", 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056371399999999997 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.856 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.883324+0000", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8129999999999998e-06 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.888364+0000", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0050400419999999998 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.897516+0000", 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0091526249999999993 2024-09-13T15:10:58.857 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.897530+0000", 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3417e-05 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.959867+0000", 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.062337203 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.858 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.959914+0000", 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7468e-05 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168490 14.2 14:4aab50f2:::benchmark_data_smithi042_319156_object44533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.923564+0000", 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.80958813000001, 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.04592503, 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.859 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168490 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923564+0000", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923564+0000", 2024-09-13T15:10:58.860 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923563+0000", 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.861 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923570+0000", 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4860000000000002e-06 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923585+0000", 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5472e-05 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923592+0000", 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5599999999999999e-06 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.862 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923629+0000", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7768000000000002e-05 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.923639+0000", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1740000000000006e-06 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.960273+0000", 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.036634005 2024-09-13T15:10:58.863 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.960359+0000", 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.6244999999999998e-05 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.960438+0000", 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9418000000000004e-05 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.864 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969300+0000", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0088615299999999994 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969343+0000", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2926000000000002e-05 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969425+0000", 2024-09-13T15:10:58.865 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2640999999999997e-05 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969463+0000", 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7793999999999998e-05 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969489+0000", 2024-09-13T15:10:58.866 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.6322e-05 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168505 14.2 14:44e83c52:::benchmark_data_smithi042_319156_object44548:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.936080+0000", 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.797072703, 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0387772799999999, 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.868 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168505 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936080+0000", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.869 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936080+0000", 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936078+0000", 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936085+0000", 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8619999999999997e-06 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.870 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936098+0000", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3039e-05 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936105+0000", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7719999999999997e-06 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936148+0000", 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2923999999999998e-05 2024-09-13T15:10:58.871 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936157+0000", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0110000000000006e-06 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.963282+0000", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.027125472 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.963381+0000", 2024-09-13T15:10:58.872 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.8884999999999999e-05 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.963461+0000", 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9583e-05 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969543+0000", 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0060817919999999999 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.873 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969595+0000", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2089999999999998e-05 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974776+0000", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0051812259999999997 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974821+0000", 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5398999999999997e-05 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.874 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974857+0000", 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5682000000000002e-05 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168506 14.2 14:495c6dee:::benchmark_data_smithi042_319156_object44549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.936994+0000", 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.79615849499999, 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.038583458, 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.875 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168506 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936994+0000", 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.876 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936994+0000", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936993+0000", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.936998+0000", 2024-09-13T15:10:58.877 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3890000000000004e-06 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.937009+0000", 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0355e-05 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.937015+0000", 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1750000000000002e-06 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.878 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.937052+0000", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7123999999999999e-05 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.937060+0000", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7749999999999993e-06 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.966361+0000", 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0293010339999999 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.879 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.966468+0000", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000107588 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.966585+0000", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000116907 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969586+0000", 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0030010710000000001 2024-09-13T15:10:58.880 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969602+0000", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.5557e-05 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974961+0000", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053593180000000001 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975002+0000", 2024-09-13T15:10:58.881 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0915999999999999e-05 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975577+0000", 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00057529100000000002 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168515 14.2 14:4cc2bd27:::benchmark_data_smithi042_319156_object44558:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.950439+0000", 2024-09-13T15:10:58.882 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.782713871, 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.03197388, 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168515 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950439+0000", 2024-09-13T15:10:58.883 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950439+0000", 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950438+0000", 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.884 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950442+0000", 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6920000000000001e-06 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950455+0000", 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.29e-05 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950462+0000", 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0269999999999998e-06 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.885 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950593+0000", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013035799999999999 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.950601+0000", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8200000000000003e-06 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969629+0000", 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.019027092 2024-09-13T15:10:58.886 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969702+0000", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3456999999999998e-05 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.969790+0000", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8490000000000001e-05 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974911+0000", 2024-09-13T15:10:58.887 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0051207020000000004 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.974921+0000", 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4870000000000008e-06 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982290+0000", 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0073694859999999997 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.888 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982376+0000", 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.5827000000000003e-05 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982413+0000", 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6520000000000003e-05 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168525 14.2 14:47f8e3cf:::benchmark_data_smithi042_319156_object44568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.961120+0000", 2024-09-13T15:10:58.889 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.77203222700001, 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0340342229999999, 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168525 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961120+0000", 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.890 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961120+0000", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961119+0000", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961125+0000", 2024-09-13T15:10:58.891 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4840000000000001e-06 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961140+0000", 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4218e-05 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961146+0000", 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6939999999999998e-06 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.892 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961190+0000", 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3205999999999999e-05 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.961197+0000", 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9699999999999999e-06 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.972187+0000", 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0109891449999999 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.893 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.972257+0000", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9962999999999994e-05 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.972323+0000", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6585e-05 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975632+0000", 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0033084880000000001 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.894 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982455+0000", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0068235980000000002 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995088+0000", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012632512 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995116+0000", 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.87e-05 2024-09-13T15:10:58.895 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995155+0000", 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8019000000000003e-05 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168534 14.2 14:4a67fd52:::benchmark_data_smithi042_319156_object44577:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.973731+0000", 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.75942126199999, 2024-09-13T15:10:58.896 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.021553079, 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168534 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973731+0000", 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.897 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973731+0000", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973729+0000", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973738+0000", 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.2500000000000006e-06 2024-09-13T15:10:58.898 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973751+0000", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3689000000000001e-05 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973758+0000", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8870000000000003e-06 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973807+0000", 2024-09-13T15:10:58.899 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8560000000000003e-05 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.973816+0000", 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.465e-06 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975656+0000", 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0018395630000001 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.900 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975777+0000", 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000121101 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.975895+0000", 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000117558 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988730+0000", 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012835447 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.901 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988758+0000", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.8498999999999999e-05 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995195+0000", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0064369090000000002 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995247+0000", 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1465000000000003e-05 2024-09-13T15:10:58.902 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995284+0000", 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.7455e-05 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168543 14.2 14:46792d06:::benchmark_data_smithi042_319156_object44586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:47.985144+0000", 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.74800835799999, 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.010221909, 2024-09-13T15:10:58.903 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168543 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985144+0000", 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.904 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985144+0000", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985142+0000", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985150+0000", 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9780000000000005e-06 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.905 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985168+0000", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7754e-05 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985175+0000", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4919999999999996e-06 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985227+0000", 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.1727000000000001e-05 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.906 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:47.985236+0000", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1160000000000005e-06 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.978917+0000", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99368114100000005 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.979028+0000", 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000110142 2024-09-13T15:10:58.907 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.979144+0000", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000116736 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988786+0000", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0096412900000000003 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988813+0000", 2024-09-13T15:10:58.908 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.7285e-05 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995293+0000", 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.006480375 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995333+0000", 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9702e-05 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.909 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995366+0000", 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3154000000000001e-05 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168605 14.2 14:48078ac1:::benchmark_data_smithi042_319156_object44648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:48.052648+0000", 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.68050427700001, 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.94279482800000003, 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.910 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168605 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052648+0000", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.911 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052648+0000", 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052647+0000", 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052652+0000", 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.3240000000000002e-06 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.912 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052663+0000", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0589e-05 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052668+0000", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2510000000000002e-06 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052716+0000", 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7800999999999999e-05 2024-09-13T15:10:58.913 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.052726+0000", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4599999999999992e-06 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982477+0000", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.92975177499999995 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982583+0000", 2024-09-13T15:10:58.914 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00010582999999999999 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.982695+0000", 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011160200000000001 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995077+0000", 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012382389000000001 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.915 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995190+0000", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011244899999999999 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995373+0000", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00018319000000000001 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995413+0000", 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9981999999999998e-05 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.916 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995443+0000", 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0246000000000001e-05 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:168621 14.2 14:4c4d9156:::benchmark_data_smithi042_319156_object44664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:48.075933+0000", 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 190.657219355, 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.92598742099999998, 2024-09-13T15:10:58.917 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 168621 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075933+0000", 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.918 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075933+0000", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075932+0000", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075937+0000", 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5680000000000001e-06 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.919 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075947+0000", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6739999999999992e-06 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075952+0000", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5169999999999999e-06 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075978+0000", 2024-09-13T15:10:58.920 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.5409999999999999e-05 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.075981+0000", 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8020000000000002e-06 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988837+0000", 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.91285602200000004 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.921 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.988928+0000", 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0177000000000001e-05 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.989025+0000", 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.7942000000000005e-05 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995454+0000", 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0064289330000000004 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.922 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:48.995670+0000", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00021595800000000001 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:49.001804+0000", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0061335920000000002 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:49.001884+0000", 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0500000000000005e-05 2024-09-13T15:10:58.923 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:49.001921+0000", 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6117e-05 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:171292 14.2 14:4ef54ca4:::benchmark_data_smithi042_319156_object47335:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:51.893972+0000", 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 186.839180545, 2024-09-13T15:10:58.924 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.183675246, 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 171292 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893972+0000", 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.925 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893972+0000", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893971+0000", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893976+0000", 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6409999999999999e-06 2024-09-13T15:10:58.926 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893987+0000", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0648e-05 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.893994+0000", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8170000000000001e-06 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.894037+0000", 2024-09-13T15:10:58.927 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2691000000000002e-05 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.894172+0000", 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000135418 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.894304+0000", 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00013197 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.928 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.906625+0000", 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012321016000000001 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.908728+0000", 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0021026080000000002 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.914013+0000", 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0052850199999999996 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.929 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.914026+0000", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3627e-05 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919850+0000", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0058236149999999 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919862+0000", 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1819e-05 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.930 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077586+0000", 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1577239500000001 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077647+0000", 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1622000000000001e-05 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.931 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:171299 14.2 14:49b9fb6c:::benchmark_data_smithi042_319156_object47342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:51.895433+0000", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 186.83771920199999, 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.1824484379999998, 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "reached pg", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 171299 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.932 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895433+0000", 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895433+0000", 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895433+0000", 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.933 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895436+0000", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.179e-06 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895445+0000", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.7299999999999994e-06 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.895451+0000", 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2670000000000003e-06 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.934 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.897451+0000", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.001999645 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.897583+0000", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000132638 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 3,5", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.897724+0000", 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00014045799999999999 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.935 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.908718+0000", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.010994467000000001 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.908736+0000", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7742999999999999e-05 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.914056+0000", 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053202550000000003 2024-09-13T15:10:58.936 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:51.914065+0000", 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0710000000000001e-06 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919889+0000", 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0058240620000001 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.937 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919897+0000", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9449999999999993e-06 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077838+0000", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.157940744 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077882+0000", 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3767999999999998e-05 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.938 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:171345 14.2 14:4a8849aa:::benchmark_data_smithi042_319156_object47388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:52.021622+0000", 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 186.71153011999999, 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.062476427, 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 171345 2024-09-13T15:10:58.939 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021622+0000", 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021622+0000", 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.940 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021620+0000", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999981 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021630+0000", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.5389999999999996e-06 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021647+0000", 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6833e-05 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.941 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021655+0000", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0600000000000008e-06 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021725+0000", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.0456000000000004e-05 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.021737+0000", 2024-09-13T15:10:58.942 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1800000000000001e-05 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919987+0000", 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.89825001000000004 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:52.919996+0000", 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.8000000000000004e-06 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.943 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077909+0000", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.157912746 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.077986+0000", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7650000000000004e-05 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3,5", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.078061+0000", 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.4987999999999994e-05 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.944 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.083926+0000", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0058649990000000001 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.083972+0000", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5914999999999997e-05 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.083984+0000", 2024-09-13T15:10:58.945 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1558e-05 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.084013+0000", 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9652999999999999e-05 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.084043+0000", 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.904e-05 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.946 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:54.084099+0000", 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6298000000000002e-05 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174240 14.2 14:49efb381:::benchmark_data_smithi042_319156_object50283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.146360+0000", 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.58679217900001, 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0187001680000001, 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.947 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174240 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146360+0000", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146360+0000", 2024-09-13T15:10:58.948 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146359+0000", 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146365+0000", 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0419999999999999e-06 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.949 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146378+0000", 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3318e-05 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146385+0000", 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9330000000000004e-06 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146440+0000", 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4642000000000003e-05 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.950 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.146449+0000", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.6439999999999995e-06 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.150970+0000", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0045206019999999 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.151084+0000", 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000113508 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.951 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.151157+0000", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.3425999999999998e-05 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.156477+0000", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053205379999999997 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.159274+0000", 2024-09-13T15:10:58.952 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0027960680000000001 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.164992+0000", 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057187799999999997 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.165022+0000", 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.932e-05 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.953 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.165060+0000", 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8847999999999998e-05 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174247 14.2 14:4d95870f:::benchmark_data_smithi042_319156_object50290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.153416+0000", 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.579736553, 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0177014280000001, 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.954 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174247 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153416+0000", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153416+0000", 2024-09-13T15:10:58.955 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153415+0000", 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153420+0000", 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.5799999999999999e-06 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.956 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153428+0000", 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0499999999999992e-06 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153433+0000", 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8910000000000003e-06 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153482+0000", 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.888e-05 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.957 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153491+0000", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.1759999999999999e-06 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.153515+0000", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0000233059999999 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.153594+0000", 2024-09-13T15:10:58.958 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9111999999999997e-05 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.153654+0000", 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.0519000000000001e-05 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.159344+0000", 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056896890000000004 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.959 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.165117+0000", 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0057726690000000002 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171064+0000", 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0059478990000000004 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171095+0000", 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.0889999999999997e-05 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.960 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171117+0000", 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2062999999999999e-05 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174248 14.2 14:42e1e344:::benchmark_data_smithi042_319156_object50291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.153453+0000", 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.57969975699999, 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.017818015, 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.961 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174248 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153453+0000", 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.963 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153453+0000", 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153452+0000", 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.964 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153456+0000", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3139999999999999e-06 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153471+0000", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.535e-05 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153480+0000", 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.4259999999999992e-06 2024-09-13T15:10:58.965 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153519+0000", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9070999999999999e-05 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.153526+0000", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1199999999999999e-06 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.155140+0000", 2024-09-13T15:10:58.966 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.001614271 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.155220+0000", 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.0566000000000001e-05 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.155288+0000", 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7386999999999997e-05 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.967 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.162044+0000", 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0067560349999999996 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171144+0000", 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0090999919999999995 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171186+0000", 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2092000000000003e-05 2024-09-13T15:10:58.968 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171232+0000", 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6458000000000001e-05 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171271+0000", 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8447e-05 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.969 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174275 14.2 14:4d3786d8:::benchmark_data_smithi042_319156_object50318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.178146+0000", 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.555006867, 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.99881564499999997, 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174275 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.970 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178146+0000", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178146+0000", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178145+0000", 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.971 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178149+0000", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.6430000000000003e-06 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178160+0000", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0125e-05 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178164+0000", 2024-09-13T15:10:58.972 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7690000000000004e-06 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178206+0000", 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.1961e-05 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178213+0000", 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8079999999999999e-06 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.973 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.156506+0000", 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97829280299999999 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.156597+0000", 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0780999999999997e-05 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.156662+0000", 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5273999999999999e-05 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.974 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.162052+0000", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0053898369999999998 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.171329+0000", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0092771799999999995 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.176875+0000", 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.005545655 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.975 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.176923+0000", 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.8109000000000003e-05 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.176961+0000", 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.8537000000000001e-05 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.976 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174276 14.2 14:47659f26:::benchmark_data_smithi042_319156_object50319:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.178694+0000", 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.55445803800001, 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0086147089999999, 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174276 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.977 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178694+0000", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178694+0000", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178694+0000", 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.978 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178698+0000", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2769999999999999e-06 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178710+0000", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2551000000000001e-05 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178716+0000", 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2440000000000001e-06 2024-09-13T15:10:58.979 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178756+0000", 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.0259999999999997e-05 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.178764+0000", 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6299999999999998e-06 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.980 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.157921+0000", 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.97915713800000004 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.157988+0000", 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7268999999999996e-05 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.158048+0000", 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9636999999999998e-05 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.981 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.164956+0000", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0069086829999999997 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.176998+0000", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012041911000000001 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187216+0000", 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.01021735 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.982 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187266+0000", 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.0540000000000001e-05 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187309+0000", 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3028999999999997e-05 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.983 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174330 14.2 14:458ec8f8:::benchmark_data_smithi042_319156_object50373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.235738+0000", 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.49741438500001, 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.95170993400000004, 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174330 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.984 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235738+0000", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235738+0000", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235737+0000", 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.985 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235743+0000", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6109999999999999e-06 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235755+0000", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1549000000000001e-05 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235760+0000", 2024-09-13T15:10:58.986 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.2780000000000001e-06 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235805+0000", 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4363000000000002e-05 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.235811+0000", 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5980000000000002e-06 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.987 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.159353+0000", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.92354219000000004 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.159431+0000", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7562e-05 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.159544+0000", 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00011315300000000001 2024-09-13T15:10:58.988 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.164962+0000", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0054179730000000004 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187177+0000", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022215068000000001 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187381+0000", 2024-09-13T15:10:58.989 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000203969 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187416+0000", 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5380999999999999e-05 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187448+0000", 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.1569000000000002e-05 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.990 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_op(client.36129.0:174355 14.2 14:455a4e60:::benchmark_data_smithi042_319156_object50398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e427)", 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:58.259785+0000", 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 180.473367994, 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.927794703, 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "client_info": { 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "client": "client.36129", 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "client_addr": "172.21.15.42:0/327952600", 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "tid": 174355 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.991 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259785+0000", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259785+0000", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259783+0000", 2024-09-13T15:10:58.992 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259789+0000", 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9390000000000002e-06 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259801+0000", 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1552999999999999e-05 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:10:58.993 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259806+0000", 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.9110000000000001e-06 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259842+0000", 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6585000000000003e-05 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for peered", 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:58.259850+0000", 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.5100000000000001e-06 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.994 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.160723+0000", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.90087357899999998 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.160807+0000", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.3598999999999997e-05 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "waiting for subops from 1,3", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.160873+0000", 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5773999999999997e-05 2024-09-13T15:10:58.995 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "op_commit", 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.165073+0000", 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0041998729999999998 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187350+0000", 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.022277469000000001 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:10:58.996 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187527+0000", 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000177462 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187561+0000", 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.3618000000000002e-05 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.187579+0000", 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8121000000000001e-05 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.997 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.998 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.998 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:10:58.998 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:10:58.998 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:10:59.027 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.6 to 1 (10000) 2024-09-13T15:10:59.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:10:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:10:59.099 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:10:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:10:59.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:10:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:10:59.815 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-13T15:10:59.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:10:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:10:59] "GET /metrics HTTP/1.1" 200 34119 "" "Prometheus/2.51.0" 2024-09-13T15:11:00.219 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[113926]: pgmap v1683: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:00.219 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:00.219 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[113926]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T15:11:00.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[108570]: pgmap v1683: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:00.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:00.220 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:00 smithi042 ceph-mon[108570]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T15:11:00.244 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-13T15:11:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:00 smithi159 ceph-mon[102519]: pgmap v1683: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:00 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/387289121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:00.249 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:00 smithi159 ceph-mon[102519]: osdmap e487: 8 total, 8 up, 8 in 2024-09-13T15:11:00.256 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:11:00.793 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:02.209 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:02 smithi159 ceph-mon[102519]: pgmap v1685: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:02.209 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:11:02.210 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:11:02.210 INFO:teuthology.orchestra.run.smithi159.stdout: "num_ops": 0 2024-09-13T15:11:02.210 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:11:02.332 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:02 smithi042 ceph-mon[113926]: pgmap v1685: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:02.333 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:02 smithi042 ceph-mon[108570]: pgmap v1685: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:03.390 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:03 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:03.390 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:03 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:03.414 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-13T15:11:03.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:03.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:03 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:03.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:03 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.257 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:11:04.275 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-13T15:11:04.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:04 smithi159 ceph-mon[102519]: pgmap v1686: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:04.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:04 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.529 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:04 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.530 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:04 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:11:04.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[113926]: pgmap v1686: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:04.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.587 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:11:04.587 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[108570]: pgmap v1686: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:04.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:04.588 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:04 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-13T15:11:04.951 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-13T15:11:05.393 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:05.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:05 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:05.550 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:05 smithi042 ceph-mon[108570]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T15:11:05.550 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:05 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:05.550 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:05 smithi042 ceph-mon[113926]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T15:11:05.561 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:05 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1783847223' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-13T15:11:05.561 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:05 smithi159 ceph-mon[102519]: osdmap e488: 8 total, 8 up, 8 in 2024-09-13T15:11:06.414 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:06 smithi042 ceph-mon[113926]: pgmap v1688: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:06.414 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:06.414 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:06 smithi042 ceph-mon[108570]: pgmap v1688: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:06.415 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:06.417 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:11:06.417 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [], 2024-09-13T15:11:06.417 INFO:teuthology.orchestra.run.smithi159.stdout: "complaint_time": 30, 2024-09-13T15:11:06.418 INFO:teuthology.orchestra.run.smithi159.stdout: "num_blocked_ops": 0 2024-09-13T15:11:06.418 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:11:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:06 smithi159 ceph-mon[102519]: pgmap v1688: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:07.064 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-13T15:11:07.286 INFO:teuthology.orchestra.run.smithi159.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T15:11:08.446 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:08 smithi159 ceph-mon[102519]: pgmap v1689: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:08.446 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:08.447 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:08 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:08.448 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[108570]: pgmap v1689: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:08.449 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:08.449 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:08.449 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[113926]: pgmap v1689: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:08.449 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:08.449 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:08 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:09.022 INFO:teuthology.orchestra.run.smithi159.stdout:{ 2024-09-13T15:11:09.022 INFO:teuthology.orchestra.run.smithi159.stdout: "size": 20, 2024-09-13T15:11:09.022 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 600, 2024-09-13T15:11:09.022 INFO:teuthology.orchestra.run.smithi159.stdout: "ops": [ 2024-09-13T15:11:09.022 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110697 13.f e361/344 13:f677430e:::benchmark_data_smithi042_286876_object110696:head v 361'6864, mlcod=361'6862)", 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 529.44831752899995, 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33391684900000002, 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.023 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571582+0000", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571581+0000", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571627+0000", 2024-09-13T15:11:09.024 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5974999999999999e-05 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571640+0000", 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2938e-05 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571647+0000", 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7179999999999999e-06 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.025 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571691+0000", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.4731999999999998e-05 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.571711+0000", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.0064e-05 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905463+0000", 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33375128300000001 2024-09-13T15:11:09.026 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905499+0000", 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6229000000000002e-05 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.027 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110698 13.d e361/344 13:b5e3a503:::benchmark_data_smithi042_286876_object110697:head v 361'6993, mlcod=361'6991)", 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 529.44760978700003, 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34348563700000001, 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.028 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572290+0000", 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572289+0000", 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.029 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572332+0000", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2815000000000003e-05 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572343+0000", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1163e-05 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572349+0000", 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3430000000000001e-06 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.030 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572404+0000", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.4199000000000001e-05 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.572422+0000", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8601999999999999e-05 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.915732+0000", 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.34331013199999999 2024-09-13T15:11:09.031 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.915775+0000", 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.3176999999999997e-05 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:110704 13.e e361/354 13:793880cd:::benchmark_data_smithi042_286876_object110703:head v 361'6942, mlcod=361'6940)", 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 529.44566133299998, 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33492928500000002, 2024-09-13T15:11:09.032 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574238+0000", 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.033 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574237+0000", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.574388+0000", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000150798 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905431+0000", 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.33104323499999999 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.034 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905444+0000", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.2891000000000001e-05 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905490+0000", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.613e-05 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.905526+0000", 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.5531000000000003e-05 2024-09-13T15:11:09.035 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.909089+0000", 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0035635139999999998 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:19.909168+0000", 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8368000000000006e-05 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.036 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:135784 13.f e366/344 13:fb444c05:::benchmark_data_smithi042_286876_object12283:head v 366'8466, mlcod=366'8463)", 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 502.29551482099998, 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040515675000000001, 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:11:09.037 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724385+0000", 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724384+0000", 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.038 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724390+0000", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6509999999999998e-06 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724402+0000", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1885999999999999e-05 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.724408+0000", 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.2790000000000004e-06 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.039 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.730074+0000", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0056665070000000003 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.730093+0000", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8859999999999999e-05 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.764858+0000", 2024-09-13T15:11:09.040 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034764694999999998 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:02:46.764900+0000", 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2410000000000002e-05 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237925 13.7 e387/344 13:e4e8a23a:::benchmark_data_smithi042_286876_object114424:head v 387'14553, mlcod=387'14549)", 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:11:09.041 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.95420879900001, 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.041768151000000003, 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.042 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065691+0000", 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065690+0000", 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065696+0000", 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4470000000000003e-06 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.043 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065713+0000", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6371e-05 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.065720+0000", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9550000000000003e-06 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066779+0000", 2024-09-13T15:11:09.044 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0010590269999999999 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066835+0000", 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.647e-05 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107422+0000", 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040587217000000002 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.045 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107459+0000", 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6520999999999998e-05 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237928 13.7 e387/344 13:e0e58808:::benchmark_data_smithi042_286876_object114427:head v 387'14554, mlcod=387'14550)", 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.953404283, 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040986935000000002, 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.046 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066495+0000", 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.047 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066501+0000", 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9719999999999997e-06 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066524+0000", 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.3447000000000001e-05 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.048 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.066531+0000", 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1810000000000004e-06 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.071239+0000", 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0047075370000000004 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.071304+0000", 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5053000000000002e-05 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.049 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107465+0000", 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.036161445 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107482+0000", 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7028999999999999e-05 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237932 13.f e387/344 13:f7f5141f:::benchmark_data_smithi042_286876_object114431:head v 387'14815, mlcod=387'14813)", 2024-09-13T15:11:09.050 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.95162399999998, 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.039224614999999997, 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.051 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068276+0000", 2024-09-13T15:11:09.052 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.052 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.052 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068275+0000", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068281+0000", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5810000000000002e-06 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068296+0000", 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.4861000000000001e-05 2024-09-13T15:11:09.053 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.068303+0000", 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.579e-06 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.072526+0000", 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0042231179999999997 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.054 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.072584+0000", 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.7729000000000002e-05 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107484+0000", 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.034900333999999998 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.107500+0000", 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.6285000000000001e-05 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.055 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237933 13.f e387/344 13:f3c23bc3:::benchmark_data_smithi042_286876_object114432:head v 387'14816, mlcod=387'14813)", 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.94242831700001, 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.039545807000000002, 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:11:09.056 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077471+0000", 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077470+0000", 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.057 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077477+0000", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7059999999999998e-06 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077487+0000", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0474e-05 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.077493+0000", 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1489999999999999e-06 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.058 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079962+0000", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0024686460000000001 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.080007+0000", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.5399999999999999e-05 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.111523+0000", 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.031515504 2024-09-13T15:11:09.059 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117017+0000", 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0054940689999999999 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237934 13.f e387/344 13:f0a0c5a8:::benchmark_data_smithi042_286876_object114433:head v 387'14817, mlcod=387'14813)", 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.940693452, 2024-09-13T15:11:09.060 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037943330999999997, 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079206+0000", 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.061 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079205+0000", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079212+0000", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.8920000000000003e-06 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079458+0000", 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00024603299999999999 2024-09-13T15:11:09.062 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079465+0000", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.1790000000000002e-06 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.087686+0000", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.008220158 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.087763+0000", 2024-09-13T15:11:09.063 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.7601000000000005e-05 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117024+0000", 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.029260699000000001 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117149+0000", 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.00012558900000000001 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.064 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237942 13.7 e387/344 13:eb3314b0:::benchmark_data_smithi042_286876_object114441:head v 387'14555, mlcod=387'14551)", 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.94042740200001, 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037819001999999997, 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.065 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079472+0000", 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.066 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079476+0000", 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.7060000000000003e-06 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079483+0000", 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.4760000000000003e-06 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.079487+0000", 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.9659999999999998e-06 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.067 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.091989+0000", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.012501945 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.092056+0000", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.6649999999999994e-05 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117155+0000", 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025099082000000002 2024-09-13T15:11:09.068 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.117291+0000", 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.000136549 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:237950 13.7 e387/344 13:ee7c64af:::benchmark_data_smithi042_286876_object114449:head v 387'14556, mlcod=387'14551)", 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 387.93569714900002, 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037050540999999999, 2024-09-13T15:11:09.069 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.070 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084202+0000", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084208+0000", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.891e-06 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084218+0000", 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.0497e-05 2024-09-13T15:11:09.071 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.084224+0000", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.9510000000000003e-06 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.103717+0000", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.019492651999999999 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.103736+0000", 2024-09-13T15:11:09.072 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9004e-05 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.121185+0000", 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.017449479 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:41.121253+0000", 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.7773000000000003e-05 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.073 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:242882 13.f e389/344 13:fe304d65:::benchmark_data_smithi042_286876_object119381:head v 389'15121, mlcod=389'15119)", 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 382.69446914999997, 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.045855901999999997, 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.074 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325430+0000", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325436+0000", 2024-09-13T15:11:09.075 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3350000000000003e-06 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325450+0000", 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.3821e-05 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.325456+0000", 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.5350000000000001e-06 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.076 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.327766+0000", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0023101490000000001 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.327836+0000", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.9504000000000004e-05 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371244+0000", 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.043408081000000001 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.077 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371286+0000", 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4.2271999999999998e-05 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.45788.0:242886 13.f e389/344 13:fed0232b:::benchmark_data_smithi042_286876_object119385:head v 389'15122, mlcod=389'15119)", 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 382.68911502600002, 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.040524537999999999, 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.078 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330785+0000", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.079 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330784+0000", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999995 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330790+0000", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.3239999999999999e-06 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330802+0000", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.1375999999999999e-05 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.080 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.330808+0000", 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 6.1249999999999998e-06 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.334018+0000", 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0032106410000000002 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.334041+0000", 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.2461000000000001e-05 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.081 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371291+0000", 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.037250162000000003 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:04:46.371309+0000", 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.8139999999999999e-05 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:175101 14.7 e428/395 14:e5c03ac2:::benchmark_data_smithi042_319156_object51144:head v 428'10908, mlcod=428'10904)", 2024-09-13T15:11:09.082 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:59.734714+0000", 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 189.28518528999999, 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.025891204000000001, 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734714+0000", 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.083 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734714+0000", 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734713+0000", 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.9999986 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734721+0000", 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 8.1380000000000007e-06 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.084 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734739+0000", 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.7689999999999998e-05 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.734746+0000", 2024-09-13T15:11:09.085 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.6420000000000007e-06 2024-09-13T15:11:09.088 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.088 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.088 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.088 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.739094+0000", 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.0043476679999999998 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.739150+0000", 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 5.6073000000000003e-05 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.760527+0000", 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.021376991000000001 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.089 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.760605+0000", 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.8477000000000001e-05 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "description": "osd_repop(client.36129.0:175104 14.7 e428/395 14:e9d73f2e:::benchmark_data_smithi042_319156_object51147:head v 428'10909, mlcod=428'10904)", 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "initiated_at": "2024-09-13T15:07:59.738908+0000", 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "age": 189.28099185799999, 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.027423488999999999, 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "type_data": { 2024-09-13T15:11:09.090 INFO:teuthology.orchestra.run.smithi159.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "events": [ 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "initiated", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738908+0000", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "throttled", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738908+0000", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "header_read", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738907+0000", 2024-09-13T15:11:09.091 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 4294967295.999999 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "all_read", 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738916+0000", 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 9.0149999999999992e-06 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "dispatched", 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738935+0000", 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 1.9289000000000001e-05 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.092 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "queued_for_pg", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.738943+0000", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 7.9330000000000001e-06 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "reached_pg", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.742739+0000", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.003796191 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "started", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.742768+0000", 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 2.9237000000000001e-05 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.093 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "commit_sent", 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.766294+0000", 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 0.023526083 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: }, 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: { 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "event": "done", 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "time": "2024-09-13T15:07:59.766331+0000", 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: "duration": 3.6850999999999997e-05 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: } 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout: ] 2024-09-13T15:11:09.094 INFO:teuthology.orchestra.run.smithi159.stdout:} 2024-09-13T15:11:09.147 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.7 to 1 (10000) 2024-09-13T15:11:09.388 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/259401773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.388 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:09 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.579 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/259401773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.579 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:09 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.579 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/259401773' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.579 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:09 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:09.849 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-13T15:11:09.849 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:09] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T15:11:10.246 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-13T15:11:10.402 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: pgmap v1690: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:10.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.532 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: pgmap v1690: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: pgmap v1690: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:10.533 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: osdmap e489: 8 total, 8 up, 8 in 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T15:11:10.534 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:11:11.135 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:12.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:12 smithi042 ceph-mon[108570]: pgmap v1692: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:12.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:12 smithi042 ceph-mon[113926]: pgmap v1692: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:12 smithi159 ceph-mon[102519]: pgmap v1692: 83 pgs: 83 active+clean; 582 KiB data, 9.3 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:13.826 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:11:13.826 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:11:13.827 INFO:teuthology.orchestra.run.smithi042.stdout: "num_ops": 0 2024-09-13T15:11:13.827 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:11:14.153 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-13T15:11:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:14 smithi159 ceph-mon[102519]: pgmap v1693: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:14 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3685272171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:14 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[108570]: pgmap v1693: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3685272171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[113926]: pgmap v1693: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3685272171' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.529 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:14 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-13T15:11:14.966 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-13T15:11:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:15.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[108570]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T15:11:15.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[108570]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T15:11:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[113926]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T15:11:15.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:15 smithi042 ceph-mon[113926]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T15:11:15.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-13T15:11:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:15 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-13T15:11:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:15 smithi159 ceph-mon[102519]: osdmap e490: 8 total, 8 up, 8 in 2024-09-13T15:11:15.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:15 smithi159 ceph-mon[102519]: osdmap e491: 8 total, 8 up, 8 in 2024-09-13T15:11:15.618 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:16.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:16 smithi042 ceph-mon[108570]: pgmap v1695: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:16.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:16 smithi042 ceph-mon[113926]: pgmap v1695: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:16.386 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:16.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:16 smithi159 ceph-mon[102519]: pgmap v1695: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:18.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:18 smithi159 ceph-mon[102519]: pgmap v1697: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:18.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:18 smithi042 ceph-mon[108570]: pgmap v1697: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:18.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:18 smithi042 ceph-mon[113926]: pgmap v1697: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:19.180 INFO:teuthology.orchestra.run.smithi042.stderr:reweighted osd.0 to 1 (10000) 2024-09-13T15:11:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:19 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1306362557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:19 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:19 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1306362557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.500 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:19 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.501 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:19 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1306362557' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.501 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:19 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:19.826 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:19] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T15:11:19.850 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-13T15:11:20.085 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:11:20.085 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [], 2024-09-13T15:11:20.086 INFO:teuthology.orchestra.run.smithi042.stdout: "complaint_time": 30, 2024-09-13T15:11:20.086 INFO:teuthology.orchestra.run.smithi042.stdout: "num_blocked_ops": 0 2024-09-13T15:11:20.086 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:11:20.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[108570]: pgmap v1698: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:20.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[108570]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T15:11:20.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[113926]: pgmap v1698: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:20.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:20.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:20 smithi042 ceph-mon[113926]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T15:11:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:20 smithi159 ceph-mon[102519]: pgmap v1698: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:20 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:20.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:20 smithi159 ceph-mon[102519]: osdmap e492: 8 total, 8 up, 8 in 2024-09-13T15:11:20.766 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-13T15:11:20.831 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:21.348 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:21 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:21.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:21 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:21 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:21.597 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:22.277 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:22 smithi042 ceph-mon[108570]: pgmap v1700: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:22.277 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:22 smithi042 ceph-mon[113926]: pgmap v1700: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:22.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:22 smithi159 ceph-mon[102519]: pgmap v1700: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:24.192 INFO:teuthology.orchestra.run.smithi042.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-13T15:11:24.289 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "size": 20, 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 600, 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "ops": [ 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55042 13.3 13:c5766ce4:::benchmark_data_smithi042_286876_object55041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 594.846782738, 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25510091000000001, 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.290 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55042 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.291 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439898+0000", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439902+0000", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1839999999999999e-06 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439931+0000", 2024-09-13T15:11:24.292 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9566000000000001e-05 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439951+0000", 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.9743999999999999e-05 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.439961+0000", 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0671e-05 2024-09-13T15:11:24.293 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440069+0000", 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000107279 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440197+0000", 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012778299999999999 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.294 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.440387+0000", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00019016000000000001 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.442194+0000", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0018073729999999999 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.445468+0000", 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0032742719999999999 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.295 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694893+0000", 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.24942428 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694950+0000", 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6963000000000003e-05 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.296 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.694999+0000", 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.9635000000000002e-05 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55050 13.3 13:c347d548:::benchmark_data_smithi042_286876_object55049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 594.84044261199995, 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.25196794900000002, 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.297 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55050 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446238+0000", 2024-09-13T15:11:24.298 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446240+0000", 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0250000000000001e-06 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446264+0000", 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3869000000000001e-05 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.299 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446279+0000", 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4280999999999999e-05 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446286+0000", 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4780000000000002e-06 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.300 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446330+0000", 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3436000000000002e-05 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446454+0000", 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012442200000000001 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.301 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.446616+0000", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000161881 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.448563+0000", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0019471880000000001 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.450965+0000", 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024016760000000002 2024-09-13T15:11:24.302 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698076+0000", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.247111365 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698159+0000", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3280000000000002e-05 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698206+0000", 2024-09-13T15:11:24.303 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7048000000000001e-05 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55064 13.3 13:c67e5ab2:::benchmark_data_smithi042_286876_object55063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:11:24.304 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 594.83053291900001, 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.24222567, 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55064 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.305 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456148+0000", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456150+0000", 2024-09-13T15:11:24.306 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6509999999999999e-06 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456171+0000", 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1084999999999999e-05 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456186+0000", 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4651000000000001e-05 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.307 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456193+0000", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2629999999999997e-06 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456252+0000", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.8833999999999999e-05 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456419+0000", 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00016782 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.308 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.456566+0000", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014662900000000001 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.458701+0000", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021344670000000001 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.459472+0000", 2024-09-13T15:11:24.309 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00077120699999999999 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698268+0000", 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23879609299999999 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698329+0000", 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1620999999999998e-05 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.310 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698374+0000", 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4348999999999998e-05 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:55075 13.3 13:ce764c83:::benchmark_data_smithi042_286876_object55074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 594.82285960499996, 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23468729499999999, 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.311 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 55075 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.312 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463821+0000", 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463823+0000", 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4160000000000001e-06 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463841+0000", 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8533999999999999e-05 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.313 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463854+0000", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2129e-05 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463882+0000", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.8167999999999999e-05 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.463938+0000", 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6517000000000001e-05 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.314 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.464028+0000", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0132999999999999e-05 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.464119+0000", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.0520999999999996e-05 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.465694+0000", 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0015750359999999999 2024-09-13T15:11:24.315 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.466445+0000", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00075109500000000002 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698425+0000", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.23198013200000001 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698464+0000", 2024-09-13T15:11:24.316 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9019e-05 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:29.698509+0000", 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4595000000000002e-05 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60151 13.9 13:924bdd2a:::benchmark_data_smithi042_286876_object60150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.430578223, 2024-09-13T15:11:24.317 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99705287799999998, 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "reached pg", 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60151 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:11:24.318 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.319 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.319 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.319 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.319 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856103+0000", 2024-09-13T15:11:24.319 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856104+0000", 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.468e-06 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856132+0000", 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7880000000000001e-05 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.320 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856144+0000", 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2063e-05 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856152+0000", 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9710000000000004e-06 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863798+0000", 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076454499999999998 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.321 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863807+0000", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6600000000000007e-06 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.852422+0000", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98861485800000004 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853156+0000", 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000733528 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.322 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60163 13.9 13:90dccfe2:::benchmark_data_smithi042_286876_object60162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.430511325, 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0136024189999999, 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60163 2024-09-13T15:11:24.323 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.324 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856170+0000", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0299999999999999e-07 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856193+0000", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3039000000000001e-05 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856202+0000", 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4919999999999993e-06 2024-09-13T15:11:24.325 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.856209+0000", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.063e-06 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863839+0000", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0076298069999999997 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.863846+0000", 2024-09-13T15:11:24.326 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9500000000000004e-06 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853617+0000", 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98977083600000004 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853839+0000", 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000222084 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.327 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.853992+0000", 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015310000000000001 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862679+0000", 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0086871080000000007 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867225+0000", 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0045461590000000001 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.328 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869604+0000", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0023790230000000001 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869720+0000", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000116223 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869772+0000", 2024-09-13T15:11:24.329 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1931999999999997e-05 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60181 13.9 13:95295340:::benchmark_data_smithi042_286876_object60180:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.41513221000002, 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99836792200000002, 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.330 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60181 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871549+0000", 2024-09-13T15:11:24.331 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871550+0000", 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4099999999999998e-07 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871574+0000", 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4311000000000002e-05 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.332 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871585+0000", 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1369000000000001e-05 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871593+0000", 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3509999999999996e-06 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.333 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.333 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[108570]: pgmap v1701: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:24.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2278715750' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:24.334 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:24.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[113926]: pgmap v1701: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:24.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2278715750' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:24.334 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:24 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871655+0000", 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.2860999999999993e-05 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871666+0000", 2024-09-13T15:11:24.335 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0478e-05 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.855776+0000", 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98411022500000001 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.855909+0000", 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013322299999999999 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.336 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.856068+0000", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015866099999999999 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867208+0000", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011140097 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867348+0000", 2024-09-13T15:11:24.337 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014019600000000001 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869817+0000", 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024688209999999999 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869887+0000", 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.9543999999999997e-05 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.338 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869917+0000", 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0043999999999999e-05 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60182 13.9 13:9ddd2598:::benchmark_data_smithi042_286876_object60181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.41505307800003, 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99860969899999996, 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.339 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60182 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.340 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871628+0000", 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871629+0000", 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7999999999999995e-07 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871660+0000", 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.0911999999999998e-05 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.341 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871669+0000", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6020000000000006e-06 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871676+0000", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.6579999999999996e-06 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871872+0000", 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000195797 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.342 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.871881+0000", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.2900000000000008e-06 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.857719+0000", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98583839399999995 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.857873+0000", 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00015369400000000001 2024-09-13T15:11:24.343 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.858020+0000", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014712699999999999 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867265+0000", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0092451479999999999 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867355+0000", 2024-09-13T15:11:24.344 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.9481000000000006e-05 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870168+0000", 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.002813505 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870198+0000", 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.9811999999999998e-05 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.345 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870238+0000", 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9598999999999998e-05 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60194 13.9 13:9e098fbd:::benchmark_data_smithi042_286876_object60193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.40711638699997, 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.99081947299999995, 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.346 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60194 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.347 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879565+0000", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879566+0000", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.3540000000000001e-06 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879587+0000", 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.0975000000000002e-05 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.348 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879602+0000", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4613999999999999e-05 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879609+0000", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3579999999999997e-06 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879655+0000", 2024-09-13T15:11:24.349 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6029000000000003e-05 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.879661+0000", 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.7790000000000001e-06 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860158+0000", 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98049688099999999 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.350 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860273+0000", 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000115148 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.860394+0000", 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012133000000000001 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867336+0000", 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069417280000000003 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.351 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867364+0000", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.7767999999999999e-05 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870310+0000", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0029460110000000001 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870341+0000", 2024-09-13T15:11:24.352 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1698e-05 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870384+0000", 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2799999999999997e-05 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60196 13.9 13:9c14fad6:::benchmark_data_smithi042_286876_object60195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.353 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.40403485599995, 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98790771499999996, 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60196 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.354 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882646+0000", 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882648+0000", 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.525e-06 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.355 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882672+0000", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.4334000000000001e-05 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882687+0000", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5045000000000001e-05 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882696+0000", 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.405e-06 2024-09-13T15:11:24.356 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882770+0000", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4103000000000005e-05 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.882782+0000", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2369e-05 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862710+0000", 2024-09-13T15:11:24.357 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979927667 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862823+0000", 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000113725 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.862977+0000", 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000153203 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.358 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867369+0000", 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.004392796 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867430+0000", 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0168999999999999e-05 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870441+0000", 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0030115739999999999 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.359 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870501+0000", 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9806000000000003e-05 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870554+0000", 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.2994000000000001e-05 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.360 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60200 13.9 13:99cafdac:::benchmark_data_smithi042_286876_object60199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.40152736300001, 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98952164600000003, 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60200 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.361 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885154+0000", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885156+0000", 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3420000000000002e-06 2024-09-13T15:11:24.362 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885179+0000", 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3436999999999999e-05 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885201+0000", 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1253000000000001e-05 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.363 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885212+0000", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0804e-05 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885289+0000", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7435000000000007e-05 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.885300+0000", 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0947999999999999e-05 2024-09-13T15:11:24.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865182+0000", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.979881954 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865299+0000", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011673200000000001 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.865433+0000", 2024-09-13T15:11:24.365 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00013482699999999999 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.869562+0000", 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0041289170000000002 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870619+0000", 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0010570009999999999 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.366 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874577+0000", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003957899 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874632+0000", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5188000000000001e-05 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874675+0000", 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2908999999999999e-05 2024-09-13T15:11:24.367 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:60202 13.9 13:9b86202e:::benchmark_data_smithi042_286876_object60201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 591.40035408599999, 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98848820400000004, 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.368 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 60202 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886327+0000", 2024-09-13T15:11:24.369 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886328+0000", 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.376e-06 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886352+0000", 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3530999999999999e-05 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.370 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886368+0000", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.6064000000000001e-05 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886377+0000", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.6859999999999995e-06 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886442+0000", 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.4979000000000002e-05 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.371 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for peered", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:32.886453+0000", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1392999999999999e-05 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867503+0000", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.98104998499999996 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867622+0000", 2024-09-13T15:11:24.372 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000118539 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 2,5", 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.867722+0000", 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000100704 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870249+0000", 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0025271690000000001 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.373 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.870906+0000", 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000656523 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874725+0000", 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0038195 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874771+0000", 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6014999999999999e-05 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.374 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:33.874815+0000", 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3739999999999998e-05 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89942 13.3 13:cd1c33de:::benchmark_data_smithi042_286876_object89941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 565.97177667200003, 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20837172000000001, 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.375 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89942 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.376 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314904+0000", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314906+0000", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2920000000000001e-06 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314928+0000", 2024-09-13T15:11:24.377 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.194e-05 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314939+0000", 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1776e-05 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.314946+0000", 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.0330000000000003e-06 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.378 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.319828+0000", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0048817699999999997 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.319962+0000", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000134227 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.320106+0000", 2024-09-13T15:11:24.379 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000144067 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.323426+0000", 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0033193440000000001 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326450+0000", 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003023759 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.380 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523109+0000", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.196659522 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523220+0000", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011048 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.523276+0000", 2024-09-13T15:11:24.381 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6509999999999999e-05 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89954 13.3 13:c3093c51:::benchmark_data_smithi042_286876_object89953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 565.96026222299997, 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.21377985499999999, 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.382 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89954 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326419+0000", 2024-09-13T15:11:24.383 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326421+0000", 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.8360000000000001e-06 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326444+0000", 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.3133999999999999e-05 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.384 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326456+0000", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2500999999999999e-05 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326464+0000", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.2370000000000003e-06 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326791+0000", 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000327917 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.385 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327063+0000", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00027153300000000002 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.327469+0000", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00040561699999999999 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334420+0000", 2024-09-13T15:11:24.386 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0069510140000000002 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334463+0000", 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3658000000000001e-05 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.539968+0000", 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20550506900000001 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.387 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.540154+0000", 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000185768 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.540199+0000", 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.4570999999999997e-05 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89957 13.3 13:c7f138f4:::benchmark_data_smithi042_286876_object89956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:11:24.388 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 565.95992670299995, 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.21694497500000001, 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89957 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.389 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326754+0000", 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326755+0000", 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0270000000000001e-06 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.390 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326778+0000", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2809000000000001e-05 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326791+0000", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2849e-05 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.326799+0000", 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.3890000000000005e-06 2024-09-13T15:11:24.391 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.330353+0000", 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0035536000000000001 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.330768+0000", 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00041488099999999997 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.392 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.331300+0000", 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00053230699999999998 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.334495+0000", 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.003195008 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.339789+0000", 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0052936379999999998 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.393 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543579+0000", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20378991099999999 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543646+0000", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.7162999999999994e-05 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543699+0000", 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.3393000000000002e-05 2024-09-13T15:11:24.394 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89967 13.3 13:c148241f:::benchmark_data_smithi042_286876_object89966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 565.94947191200004, 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20677611500000001, 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.395 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89967 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337209+0000", 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.396 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337211+0000", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.703e-06 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337232+0000", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.1271999999999999e-05 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337248+0000", 2024-09-13T15:11:24.397 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5954e-05 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337255+0000", 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.3710000000000002e-06 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337315+0000", 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.9293000000000003e-05 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.398 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337416+0000", 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010092800000000001 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.337546+0000", 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00012996199999999999 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.339985+0000", 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0024393319999999998 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.399 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.340731+0000", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00074605299999999999 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543883+0000", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20315203800000001 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543946+0000", 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.3051000000000003e-05 2024-09-13T15:11:24.400 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.543985+0000", 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.9158e-05 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:89985 13.3 13:ce68edab:::benchmark_data_smithi042_286876_object89984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 565.92451071400001, 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19764711200000001, 2024-09-13T15:11:24.401 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 89985 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.402 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362170+0000", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362172+0000", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4959999999999999e-06 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362194+0000", 2024-09-13T15:11:24.403 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 2.2582000000000002e-05 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362210+0000", 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5968999999999999e-05 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362218+0000", 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.1910000000000003e-06 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.404 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362273+0000", 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.5087999999999998e-05 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362369+0000", 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.6502999999999994e-05 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.405 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.362462+0000", 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 9.3317000000000001e-05 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.364568+0000", 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0021051519999999999 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.406 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.365147+0000", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00057914499999999999 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559557+0000", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19441054299999999 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559733+0000", 2024-09-13T15:11:24.407 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00017600499999999999 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:01:58.559817+0000", 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 8.4121000000000003e-05 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:165999 13.3 13:c1fbbb06:::benchmark_data_smithi042_286876_object42498:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:11:24.408 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 485.18072495600001, 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19437074400000001, 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 165999 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.409 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105956+0000", 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105958+0000", 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.5770000000000001e-06 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.410 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105964+0000", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.0020000000000003e-06 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105975+0000", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.1447999999999999e-05 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.105981+0000", 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 6.1090000000000003e-06 2024-09-13T15:11:24.411 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106035+0000", 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.4153999999999997e-05 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106180+0000", 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00014459400000000001 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.412 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106301+0000", 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.000120884 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300202+0000", 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19390109999999999 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300245+0000", 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.2738999999999999e-05 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.413 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300253+0000", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.9789999999999993e-06 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300284+0000", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 3.1139999999999997e-05 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300327+0000", 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.3018e-05 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.414 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:166001 13.3 13:c2d18c6b:::benchmark_data_smithi042_286876_object42500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 485.18027958200003, 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.19413623299999999, 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.415 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 166001 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106401+0000", 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.416 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106403+0000", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.15e-06 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106407+0000", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.7469999999999996e-06 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106419+0000", 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.2013e-05 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.417 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.106427+0000", 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.4410000000000004e-06 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110401+0000", 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0039738819999999998 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.418 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110506+0000", 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010556999999999999 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.110613+0000", 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00010705300000000001 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300337+0000", 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.189723578 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.419 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300384+0000", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.6999999999999997e-05 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300429+0000", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.5194999999999997e-05 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300486+0000", 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.6708e-05 2024-09-13T15:11:24.420 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300538+0000", 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 5.1895999999999999e-05 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "description": "osd_op(client.45788.0:166006 13.3 13:cae3698d:::benchmark_data_smithi042_286876_object42505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "initiated_at": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "age": 485.17774785500001, 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.20369686100000001, 2024-09-13T15:11:24.421 INFO:teuthology.orchestra.run.smithi042.stdout: "continuous": false, 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "type_data": { 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client_info": { 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client": "client.45788", 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "client_addr": "172.21.15.42:0/3198732627", 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "tid": 166006 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "events": [ 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "initiated", 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.422 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "header_read", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108933+0000", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "throttled", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108935+0000", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.311e-06 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "all_read", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108939+0000", 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8559999999999998e-06 2024-09-13T15:11:24.423 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "dispatched", 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108950+0000", 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.0804e-05 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "queued_for_pg", 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.108958+0000", 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 7.7360000000000001e-06 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "reached_pg", 2024-09-13T15:11:24.424 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115116+0000", 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.0061578730000000003 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "started", 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115226+0000", 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011048999999999999 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "waiting for subops from 3,4", 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.115338+0000", 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011165100000000001 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.425 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300614+0000", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.18527559499999999 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "op_commit", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.300628+0000", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 1.4177e-05 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "sub_op_commit_rec", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312470+0000", 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.011842455999999999 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "commit_sent", 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312581+0000", 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 0.00011113800000000001 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "event": "done", 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "time": "2024-09-13T15:03:19.312630+0000", 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: "duration": 4.8773999999999998e-05 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T15:11:24.427 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T15:11:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:24 smithi159 ceph-mon[102519]: pgmap v1701: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:24 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2278715750' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:24.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:24 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-13T15:11:25.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:25 smithi042 ceph-mon[108570]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:25.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:25 smithi042 ceph-mon[108570]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T15:11:25.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:25 smithi042 ceph-mon[113926]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:25.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:25 smithi042 ceph-mon[113926]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T15:11:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:25 smithi159 ceph-mon[102519]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-13T15:11:25.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:25 smithi159 ceph-mon[102519]: osdmap e493: 8 total, 8 up, 8 in 2024-09-13T15:11:25.586 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-13T15:11:25.586 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-13T15:11:25.586 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-13T15:11:26.069 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-13T15:11:26.070 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T15:11:26.070 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:11:26.293 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:26.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:26 smithi042 ceph-mon[108570]: pgmap v1703: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:26.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:26 smithi042 ceph-mon[113926]: pgmap v1703: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:26.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:26 smithi159 ceph-mon[102519]: pgmap v1703: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:28.242 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:11:28.243 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":493,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T15:11:24.182962+0000","last_up_change":"2024-09-13T15:00:57.172068+0000","last_in_change":"2024-09-13T15:10:03.839251+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"334","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"146","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":94,"last_clean_end":314,"up_from":348,"up_thru":477,"down_at":315,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6803","nonce":2799202115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6805","nonce":2799202115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6809","nonce":2799202115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6807","nonce":2799202115}]},"public_addr":"172.21.15.42:6803/2799202115","cluster_addr":"172.21.15.42:6805/2799202115","heartbeat_back_addr":"172.21.15.42:6809/2799202115","heartbeat_front_addr":"172.21.15.42:6807/2799202115","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":105,"up_from":109,"up_thru":474,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6811","nonce":192065102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6813","nonce":192065102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6817","nonce":192065102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6815","nonce":192065102}]},"public_addr":"172.21.15.42:6811/192065102","cluster_addr":"172.21.15.42:6813/192065102","heartbeat_back_addr":"172.21.15.42:6817/192065102","heartbeat_front_addr":"172.21.15.42:6815/192065102","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":471,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6819","nonce":3631420758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6821","nonce":3631420758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6825","nonce":3631420758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6823","nonce":3631420758}]},"public_addr":"172.21.15.42:6819/3631420758","cluster_addr":"172.21.15.42:6821/3631420758","heartbeat_back_addr":"172.21.15.42:6825/3631420758","heartbeat_front_addr":"172.21.15.42:6823/3631420758","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":127,"up_from":133,"up_thru":477,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6827","nonce":1641847048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6829","nonce":1641847048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6833","nonce":1641847048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6831","nonce":1641847048}]},"public_addr":"172.21.15.42:6827/1641847048","cluster_addr":"172.21.15.42:6829/1641847048","heartbeat_back_addr":"172.21.15.42:6833/1641847048","heartbeat_front_addr":"172.21.15.42:6831/1641847048","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":474,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":485,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":474,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":490,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":314},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":105},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.11","mappings":[{"from":3,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/2154994614":"2024-09-14T14:28:19.829303+0000","172.21.15.42:6800/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2764224015":"2024-09-14T14:28:19.829303+0000","172.21.15.159:0/2982853112":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6835/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6800/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1459960260":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/1729697261":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/471235645":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1962929896":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.159:0/1529484914":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/3438146012":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6834/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/1816541453":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.159:0/2701547919":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/3559778733":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/800406081":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.159:0/2991823830":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:11:28.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:28 smithi159 ceph-mon[102519]: pgmap v1704: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:28.590 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:28 smithi042 ceph-mon[108570]: pgmap v1704: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:28.590 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:28 smithi042 ceph-mon[113926]: pgmap v1704: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:28.938 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T15:11:28.939 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T15:11:29.145 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:29.461 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:29 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2922777699' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:29.461 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:29 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2922777699' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:29.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:29 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2922777699' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:30.064 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:29 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:29] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T15:11:30.339 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:30 smithi042 ceph-mon[113926]: pgmap v1705: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:30.339 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:30 smithi042 ceph-mon[108570]: pgmap v1705: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:30.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:30 smithi159 ceph-mon[102519]: pgmap v1705: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:31.115 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:11:31.115 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":493,"fsid":"4c2088c8-71da-11ef-bceb-c7b262605968","created":"2024-09-13T14:13:54.973587+0000","modified":"2024-09-13T15:11:24.182962+0000","last_up_change":"2024-09-13T15:00:57.172068+0000","last_in_change":"2024-09-13T15:10:03.839251+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":15,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T14:17:00.645575+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":11,"pg_placement_num":11,"pg_placement_num_target":11,"pg_num_target":11,"pg_num_pending":11,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"334","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"146","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.1800000667572021,"score_stable":2.1800000667572021,"optimal_score":1,"raw_score_acting":2.1800000667572021,"raw_score_stable":2.1800000667572021,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T14:19:26.851654+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"65","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":65,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T14:20:44.510322+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T14:20:45.216292+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0a40fd71-cbc3-4195-b082-c4f19f7f1dd4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":94,"last_clean_end":314,"up_from":348,"up_thru":477,"down_at":315,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6803","nonce":2799202115}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6805","nonce":2799202115}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6809","nonce":2799202115}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":2799202115},{"type":"v1","addr":"172.21.15.42:6807","nonce":2799202115}]},"public_addr":"172.21.15.42:6803/2799202115","cluster_addr":"172.21.15.42:6805/2799202115","heartbeat_back_addr":"172.21.15.42:6809/2799202115","heartbeat_front_addr":"172.21.15.42:6807/2799202115","state":["exists","up"]},{"osd":1,"uuid":"50b16819-9756-4dd5-957c-a2968d7f22bb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":105,"up_from":109,"up_thru":474,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6811","nonce":192065102}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6813","nonce":192065102}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6817","nonce":192065102}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":192065102},{"type":"v1","addr":"172.21.15.42:6815","nonce":192065102}]},"public_addr":"172.21.15.42:6811/192065102","cluster_addr":"172.21.15.42:6813/192065102","heartbeat_back_addr":"172.21.15.42:6817/192065102","heartbeat_front_addr":"172.21.15.42:6815/192065102","state":["exists","up"]},{"osd":2,"uuid":"de5587ea-c587-4c38-a8ce-6f2bf7f41b2d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":122,"up_from":126,"up_thru":471,"down_at":123,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6819","nonce":3631420758}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6821","nonce":3631420758}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6825","nonce":3631420758}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":3631420758},{"type":"v1","addr":"172.21.15.42:6823","nonce":3631420758}]},"public_addr":"172.21.15.42:6819/3631420758","cluster_addr":"172.21.15.42:6821/3631420758","heartbeat_back_addr":"172.21.15.42:6825/3631420758","heartbeat_front_addr":"172.21.15.42:6823/3631420758","state":["exists","up"]},{"osd":3,"uuid":"0944a1a9-ea18-4457-8d5e-cf70f7e2be0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":127,"up_from":133,"up_thru":477,"down_at":128,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6827","nonce":1641847048}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6829","nonce":1641847048}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6833","nonce":1641847048}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":1641847048},{"type":"v1","addr":"172.21.15.42:6831","nonce":1641847048}]},"public_addr":"172.21.15.42:6827/1641847048","cluster_addr":"172.21.15.42:6829/1641847048","heartbeat_back_addr":"172.21.15.42:6833/1641847048","heartbeat_front_addr":"172.21.15.42:6831/1641847048","state":["exists","up"]},{"osd":4,"uuid":"9cb88075-7a94-4890-8de9-a717d57eae24","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":474,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6800","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6801","nonce":3643931460}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6802","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6803","nonce":3643931460}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6806","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6807","nonce":3643931460}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6804","nonce":3643931460},{"type":"v1","addr":"172.21.15.159:6805","nonce":3643931460}]},"public_addr":"172.21.15.159:6801/3643931460","cluster_addr":"172.21.15.159:6803/3643931460","heartbeat_back_addr":"172.21.15.159:6807/3643931460","heartbeat_front_addr":"172.21.15.159:6805/3643931460","state":["exists","up"]},{"osd":5,"uuid":"70e09ea8-aa92-41de-9d8b-f15ebee3edac","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":485,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6808","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6809","nonce":1094422230}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6810","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6811","nonce":1094422230}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6814","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6815","nonce":1094422230}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6812","nonce":1094422230},{"type":"v1","addr":"172.21.15.159:6813","nonce":1094422230}]},"public_addr":"172.21.15.159:6809/1094422230","cluster_addr":"172.21.15.159:6811/1094422230","heartbeat_back_addr":"172.21.15.159:6815/1094422230","heartbeat_front_addr":"172.21.15.159:6813/1094422230","state":["exists","up"]},{"osd":6,"uuid":"cc9e1f9f-78f5-452a-9f78-a47340fbdefe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":474,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6816","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6817","nonce":2578924602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6818","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6819","nonce":2578924602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6822","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6823","nonce":2578924602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6820","nonce":2578924602},{"type":"v1","addr":"172.21.15.159:6821","nonce":2578924602}]},"public_addr":"172.21.15.159:6817/2578924602","cluster_addr":"172.21.15.159:6819/2578924602","heartbeat_back_addr":"172.21.15.159:6823/2578924602","heartbeat_front_addr":"172.21.15.159:6821/2578924602","state":["exists","up"]},{"osd":7,"uuid":"85448d6e-e256-43f5-aa86-bf8c47b40863","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":490,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6824","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6825","nonce":2913377602}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6826","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6827","nonce":2913377602}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6830","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6831","nonce":2913377602}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.159:6828","nonce":2913377602},{"type":"v1","addr":"172.21.15.159:6829","nonce":2913377602}]},"public_addr":"172.21.15.159:6825/2913377602","cluster_addr":"172.21.15.159:6827/2913377602","heartbeat_back_addr":"172.21.15.159:6831/2913377602","heartbeat_front_addr":"172.21.15.159:6829/2913377602","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:08.895050+0000","dead_epoch":314},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:32.167092+0000","dead_epoch":105},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:16:56.267637+0000","dead_epoch":122},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:21.491288+0000","dead_epoch":127},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:17:43.038751+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:06.975739+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:30.100694+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T14:18:53.776847+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.11","mappings":[{"from":3,"to":5}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/2154994614":"2024-09-14T14:28:19.829303+0000","172.21.15.42:6800/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2764224015":"2024-09-14T14:28:19.829303+0000","172.21.15.159:0/2982853112":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6835/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6800/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1459960260":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/2845488129":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6800/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/1729697261":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/879988818":"2024-09-14T14:14:27.026310+0000","172.21.15.42:6800/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/713801628":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/471235645":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/1561666977":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/1962929896":"2024-09-14T14:24:04.294604+0000","172.21.15.42:0/1269168698":"2024-09-14T14:14:50.147899+0000","172.21.15.159:0/1529484914":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/3438146012":"2024-09-14T14:24:53.166700+0000","172.21.15.159:6834/3376814479":"2024-09-14T14:24:53.166700+0000","172.21.15.159:0/1816541453":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/196918433":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/240011015":"2024-09-14T14:14:27.026310+0000","172.21.15.159:0/2701547919":"2024-09-14T14:24:53.166700+0000","172.21.15.42:0/3559778733":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6800/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/800406081":"2024-09-14T14:24:04.294604+0000","172.21.15.42:6801/3699801282":"2024-09-14T14:14:50.147899+0000","172.21.15.42:6801/3689509320":"2024-09-14T14:24:04.294604+0000","172.21.15.159:0/2991823830":"2024-09-14T14:24:53.166700+0000","172.21.15.42:6801/4040829401":"2024-09-14T14:19:24.781439+0000","172.21.15.42:0/4078857182":"2024-09-14T14:19:24.781439+0000","172.21.15.42:6801/1382644857":"2024-09-14T14:28:19.829303+0000","172.21.15.42:0/810688227":"2024-09-14T14:14:27.026310+0000","172.21.15.42:0/3074293401":"2024-09-14T14:14:50.147899+0000","172.21.15.42:0/3181112294":"2024-09-14T14:19:24.781439+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":15,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T15:11:31.480 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:31 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1982461701' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:31.480 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:31 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1982461701' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:31.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:31 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1982461701' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-13T15:11:31.899 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T15:11:31.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T15:11:31.900 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T15:11:31.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T15:11:31.901 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T15:11:31.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T15:11:31.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T15:11:31.902 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T15:11:32.283 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:32.308 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:32.466 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:32 smithi042 ceph-mon[108570]: pgmap v1706: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:32.466 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:32 smithi042 ceph-mon[113926]: pgmap v1706: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:32.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:32 smithi159 ceph-mon[102519]: pgmap v1706: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:34.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:34 smithi159 ceph-mon[102519]: pgmap v1707: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:34.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:34 smithi042 ceph-mon[108570]: pgmap v1707: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:34.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:34 smithi042 ceph-mon[113926]: pgmap v1707: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:34.776 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:34.776 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:34.777 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:34.777 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:35.907 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:35.908 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:36.461 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:36 smithi042 ceph-mon[108570]: pgmap v1708: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:36.461 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:36 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:36.461 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:36 smithi042 ceph-mon[113926]: pgmap v1708: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:36.461 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:36 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:36 smithi159 ceph-mon[102519]: pgmap v1708: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:36.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:36 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:38.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:38 smithi159 ceph-mon[102519]: pgmap v1709: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:38.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:38 smithi042 ceph-mon[113926]: pgmap v1709: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:38.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:38 smithi042 ceph-mon[108570]: pgmap v1709: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:38.662 INFO:teuthology.orchestra.run.smithi042.stdout:231928234619 2024-09-13T15:11:38.662 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T15:11:40.005 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:39 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:39] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T15:11:40.032 INFO:teuthology.orchestra.run.smithi042.stdout:468151435694 2024-09-13T15:11:40.032 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T15:11:40.288 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:40 smithi042 ceph-mon[113926]: pgmap v1710: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:40.288 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:40 smithi042 ceph-mon[108570]: pgmap v1710: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:40.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:40 smithi159 ceph-mon[102519]: pgmap v1710: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:42.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:42 smithi159 ceph-mon[102519]: pgmap v1711: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:42.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:42 smithi042 ceph-mon[108570]: pgmap v1711: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:42.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:42 smithi042 ceph-mon[113926]: pgmap v1711: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:43.622 INFO:teuthology.orchestra.run.smithi042.stdout:197568496257 2024-09-13T15:11:43.623 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T15:11:44.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:44 smithi042 ceph-mon[108570]: pgmap v1712: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:44.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:44 smithi042 ceph-mon[113926]: pgmap v1712: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:44.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:44 smithi159 ceph-mon[102519]: pgmap v1712: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:44.829 INFO:teuthology.orchestra.run.smithi042.stdout:1494648619139 2024-09-13T15:11:44.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T15:11:46.440 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:46 smithi042 ceph-mon[113926]: pgmap v1713: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:46.440 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:46 smithi042 ceph-mon[108570]: pgmap v1713: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:46.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:46 smithi159 ceph-mon[102519]: pgmap v1713: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:47.985 INFO:teuthology.orchestra.run.smithi042.stdout:128849019533 2024-09-13T15:11:47.986 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T15:11:48.409 INFO:teuthology.orchestra.run.smithi042.stdout:571230650745 2024-09-13T15:11:48.409 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T15:11:48.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:48 smithi159 ceph-mon[102519]: pgmap v1714: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:48.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:48 smithi042 ceph-mon[108570]: pgmap v1714: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:48.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:48 smithi042 ceph-mon[113926]: pgmap v1714: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:49.241 INFO:teuthology.orchestra.run.smithi042.stdout:541165879680 2024-09-13T15:11:49.241 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T15:11:49.925 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:49] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-09-13T15:11:50.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:50 smithi159 ceph-mon[102519]: pgmap v1715: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:50.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:50 smithi042 ceph-mon[108570]: pgmap v1715: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:50.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:50 smithi042 ceph-mon[113926]: pgmap v1715: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:50.942 INFO:teuthology.orchestra.run.smithi042.stdout:163208757897 2024-09-13T15:11:50.942 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T15:11:50.990 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:50.991 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:50.991 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:50.998 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:51.464 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:51 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:51.465 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:51 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:51.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:51 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:11:52.456 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:52 smithi042 ceph-mon[108570]: pgmap v1716: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:52.456 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:52 smithi042 ceph-mon[113926]: pgmap v1716: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:52.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:52 smithi159 ceph-mon[102519]: pgmap v1716: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:53.662 INFO:teuthology.orchestra.run.smithi042.stdout:197568496260 2024-09-13T15:11:54.389 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568496257 got 197568496260 for osd.6 2024-09-13T15:11:54.390 DEBUG:teuthology.parallel:result is None 2024-09-13T15:11:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:54 smithi159 ceph-mon[102519]: pgmap v1717: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:54.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:54 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2021361520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:11:54.518 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:54 smithi042 ceph-mon[108570]: pgmap v1717: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:54.518 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:54 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2021361520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:11:54.518 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:54 smithi042 ceph-mon[113926]: pgmap v1717: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:54.518 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:54 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2021361520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-13T15:11:56.051 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:56.052 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:56.052 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:56.077 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:11:56.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:56 smithi159 ceph-mon[102519]: pgmap v1718: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:56.505 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:56 smithi042 ceph-mon[108570]: pgmap v1718: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:56.505 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:56 smithi042 ceph-mon[113926]: pgmap v1718: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:58.400 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:58 smithi042 ceph-mon[108570]: pgmap v1719: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:58.401 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:58 smithi042 ceph-mon[113926]: pgmap v1719: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:58.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:58 smithi159 ceph-mon[102519]: pgmap v1719: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:11:59.598 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:11:59 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3286064203' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:11:59.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:11:59 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3286064203' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:11:59.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:11:59 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3286064203' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-13T15:12:00.098 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:11:59 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:11:59] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.51.0" 2024-09-13T15:12:00.543 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:00 smithi042 ceph-mon[108570]: pgmap v1720: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:00.543 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:00 smithi042 ceph-mon[113926]: pgmap v1720: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:00.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:00 smithi159 ceph-mon[102519]: pgmap v1720: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:01.485 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:01 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/2888172299' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:12:01.485 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:01 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/2888172299' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:12:01.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:01 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/2888172299' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-13T15:12:02.002 INFO:teuthology.orchestra.run.smithi042.stdout:541165879682 2024-09-13T15:12:02.596 INFO:teuthology.orchestra.run.smithi042.stdout:1494648619142 2024-09-13T15:12:02.596 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:02 smithi042 ceph-mon[113926]: pgmap v1721: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:02.597 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:02 smithi042 ceph-mon[108570]: pgmap v1721: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:02.654 INFO:teuthology.orchestra.run.smithi042.stdout:163208757899 2024-09-13T15:12:02.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:02 smithi159 ceph-mon[102519]: pgmap v1721: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:03.368 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:03 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3773029217' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:12:03.368 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:03 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3773029217' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:12:03.395 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757897 got 163208757899 for osd.5 2024-09-13T15:12:03.395 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:03.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:03 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3773029217' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-13T15:12:04.066 INFO:tasks.cephadm.ceph_manager.ceph:need seq 541165879680 got 541165879682 for osd.2 2024-09-13T15:12:04.066 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:04.301 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:04 smithi042 ceph-mon[113926]: pgmap v1722: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:04.302 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:04 smithi042 ceph-mon[108570]: pgmap v1722: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:04.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:04 smithi159 ceph-mon[102519]: pgmap v1722: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:05.768 INFO:teuthology.orchestra.run.smithi042.stdout:231928234624 2024-09-13T15:12:06.083 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1494648619139 got 1494648619142 for osd.0 2024-09-13T15:12:06.083 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:06.348 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[108570]: pgmap v1723: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3695174256' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:12:06.349 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[113926]: pgmap v1723: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3695174256' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:12:06.349 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:06 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:06.586 INFO:teuthology.orchestra.run.smithi042.stdout:468151435700 2024-09-13T15:12:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:06 smithi159 ceph-mon[102519]: pgmap v1723: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:06 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3695174256' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-13T15:12:06.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:06 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T15:12:06.923 INFO:teuthology.orchestra.run.smithi042.stdout:571230650748 2024-09-13T15:12:07.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/3456338949' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:12:07.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:07 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/1268290520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:12:07.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/3456338949' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:12:07.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:07 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/1268290520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:12:07.673 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435694 got 468151435700 for osd.1 2024-09-13T15:12:07.673 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:07.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/3456338949' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-13T15:12:07.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:07 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/1268290520' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-13T15:12:08.468 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:08 smithi042 ceph-mon[108570]: pgmap v1724: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:08.469 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:08 smithi042 ceph-mon[113926]: pgmap v1724: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:08.499 INFO:teuthology.orchestra.run.smithi042.stdout:128849019536 2024-09-13T15:12:08.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:08 smithi159 ceph-mon[102519]: pgmap v1724: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:08.846 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234619 got 231928234624 for osd.7 2024-09-13T15:12:08.846 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:09.259 INFO:tasks.cephadm.ceph_manager.ceph:need seq 571230650745 got 571230650748 for osd.3 2024-09-13T15:12:09.259 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:09.555 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:09 smithi042 ceph-mon[108570]: from='client.? 172.21.15.42:0/435731798' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:12:09.556 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:09 smithi042 ceph-mon[113926]: from='client.? 172.21.15.42:0/435731798' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:12:09.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:09 smithi159 ceph-mon[102519]: from='client.? 172.21.15.42:0/435731798' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-13T15:12:09.848 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:09 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:12:09] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-09-13T15:12:09.884 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019533 got 128849019536 for osd.4 2024-09-13T15:12:09.885 DEBUG:teuthology.parallel:result is None 2024-09-13T15:12:09.885 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-13T15:12:09.885 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:12:10.102 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:10.536 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:10 smithi159 ceph-mon[102519]: pgmap v1725: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:10.537 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:10 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:10.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:10 smithi042 ceph-mon[108570]: pgmap v1725: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:10.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:10 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:10 smithi042 ceph-mon[113926]: pgmap v1725: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:10.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:10 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T15:12:12.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:12 smithi159 ceph-mon[102519]: pgmap v1726: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:12.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:12 smithi042 ceph-mon[113926]: pgmap v1726: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:12.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:12 smithi042 ceph-mon[108570]: pgmap v1726: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:12.605 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:12:12.606 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:12:13.315 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1726,"stamp":"2024-09-13T15:12:10.836256+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":5954,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9675628,"kb_used_data":52300,"kb_used_omap":33,"kb_used_meta":9623198,"kb_avail":740285588,"statfs":{"total":767960285184,"available":758052442112,"internally_reserved":0,"allocated":53555200,"data_stored":32989154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33996,"internal_metadata":9854155572},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002971"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":851,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304357+0000","last_change":"2024-09-13T15:10:26.313323+0000","last_active":"2024-09-13T15:11:25.304357+0000","last_peered":"2024-09-13T15:11:25.304357+0000","last_clean":"2024-09-13T15:11:25.304357+0000","last_became_active":"2024-09-13T15:10:26.312986+0000","last_became_peered":"2024-09-13T15:10:26.312986+0000","last_unstale":"2024-09-13T15:11:25.304357+0000","last_undegraded":"2024-09-13T15:11:25.304357+0000","last_fullsized":"2024-09-13T15:11:25.304357+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:07.615254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":150,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418821+0000","last_change":"2024-09-13T15:11:15.162702+0000","last_active":"2024-09-13T15:11:19.418821+0000","last_peered":"2024-09-13T15:11:19.418821+0000","last_clean":"2024-09-13T15:11:19.418821+0000","last_became_active":"2024-09-13T15:11:15.162324+0000","last_became_peered":"2024-09-13T15:11:15.162324+0000","last_unstale":"2024-09-13T15:11:19.418821+0000","last_undegraded":"2024-09-13T15:11:19.418821+0000","last_fullsized":"2024-09-13T15:11:19.418821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:49.486702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":341,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281461+0000","last_change":"2024-09-13T15:10:56.023607+0000","last_active":"2024-09-13T15:11:19.281461+0000","last_peered":"2024-09-13T15:11:19.281461+0000","last_clean":"2024-09-13T15:11:19.281461+0000","last_became_active":"2024-09-13T15:10:56.023279+0000","last_became_peered":"2024-09-13T15:10:56.023279+0000","last_unstale":"2024-09-13T15:11:19.281461+0000","last_undegraded":"2024-09-13T15:11:19.281461+0000","last_fullsized":"2024-09-13T15:11:19.281461+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:57.733968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.189994+0000","last_change":"2024-09-13T15:10:04.872212+0000","last_active":"2024-09-13T15:11:24.189994+0000","last_peered":"2024-09-13T15:11:24.189994+0000","last_clean":"2024-09-13T15:11:24.189994+0000","last_became_active":"2024-09-13T15:10:04.871786+0000","last_became_peered":"2024-09-13T15:10:04.871786+0000","last_unstale":"2024-09-13T15:11:24.189994+0000","last_undegraded":"2024-09-13T15:11:24.189994+0000","last_fullsized":"2024-09-13T15:11:24.189994+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:50.179272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190612+0000","last_change":"2024-09-13T15:10:04.876599+0000","last_active":"2024-09-13T15:11:24.190612+0000","last_peered":"2024-09-13T15:11:24.190612+0000","last_clean":"2024-09-13T15:11:24.190612+0000","last_became_active":"2024-09-13T15:10:04.876243+0000","last_became_peered":"2024-09-13T15:10:04.876243+0000","last_unstale":"2024-09-13T15:11:24.190612+0000","last_undegraded":"2024-09-13T15:11:24.190612+0000","last_fullsized":"2024-09-13T15:11:24.190612+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:40.860759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418671+0000","last_change":"2024-09-13T15:11:15.162927+0000","last_active":"2024-09-13T15:11:19.418671+0000","last_peered":"2024-09-13T15:11:19.418671+0000","last_clean":"2024-09-13T15:11:19.418671+0000","last_became_active":"2024-09-13T15:11:15.162626+0000","last_became_peered":"2024-09-13T15:11:15.162626+0000","last_unstale":"2024-09-13T15:11:19.418671+0000","last_undegraded":"2024-09-13T15:11:19.418671+0000","last_fullsized":"2024-09-13T15:11:19.418671+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:36.424513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":856,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304488+0000","last_change":"2024-09-13T15:10:26.313311+0000","last_active":"2024-09-13T15:11:25.304488+0000","last_peered":"2024-09-13T15:11:25.304488+0000","last_clean":"2024-09-13T15:11:25.304488+0000","last_became_active":"2024-09-13T15:10:26.313011+0000","last_became_peered":"2024-09-13T15:10:26.313011+0000","last_unstale":"2024-09-13T15:11:25.304488+0000","last_undegraded":"2024-09-13T15:11:25.304488+0000","last_fullsized":"2024-09-13T15:11:25.304488+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:18.847442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":608,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.047050+0000","last_change":"2024-09-13T15:10:04.883100+0000","last_active":"2024-09-13T15:11:25.047050+0000","last_peered":"2024-09-13T15:11:25.047050+0000","last_clean":"2024-09-13T15:11:25.047050+0000","last_became_active":"2024-09-13T15:10:04.869463+0000","last_became_peered":"2024-09-13T15:10:04.869463+0000","last_unstale":"2024-09-13T15:11:25.047050+0000","last_undegraded":"2024-09-13T15:11:25.047050+0000","last_fullsized":"2024-09-13T15:11:25.047050+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:18.719681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.419094+0000","last_change":"2024-09-13T15:11:15.162536+0000","last_active":"2024-09-13T15:11:19.419094+0000","last_peered":"2024-09-13T15:11:19.419094+0000","last_clean":"2024-09-13T15:11:19.419094+0000","last_became_active":"2024-09-13T15:11:15.162161+0000","last_became_peered":"2024-09-13T15:11:15.162161+0000","last_unstale":"2024-09-13T15:11:19.419094+0000","last_undegraded":"2024-09-13T15:11:19.419094+0000","last_fullsized":"2024-09-13T15:11:19.419094+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:47.281436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":328,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281211+0000","last_change":"2024-09-13T15:10:56.023909+0000","last_active":"2024-09-13T15:11:19.281211+0000","last_peered":"2024-09-13T15:11:19.281211+0000","last_clean":"2024-09-13T15:11:19.281211+0000","last_became_active":"2024-09-13T15:10:56.023038+0000","last_became_peered":"2024-09-13T15:10:56.023038+0000","last_unstale":"2024-09-13T15:11:19.281211+0000","last_undegraded":"2024-09-13T15:11:19.281211+0000","last_fullsized":"2024-09-13T15:11:19.281211+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:36.486723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":136,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.419029+0000","last_change":"2024-09-13T15:11:15.162813+0000","last_active":"2024-09-13T15:11:19.419029+0000","last_peered":"2024-09-13T15:11:19.419029+0000","last_clean":"2024-09-13T15:11:19.419029+0000","last_became_active":"2024-09-13T15:11:15.162071+0000","last_became_peered":"2024-09-13T15:11:15.162071+0000","last_unstale":"2024-09-13T15:11:19.419029+0000","last_undegraded":"2024-09-13T15:11:19.419029+0000","last_fullsized":"2024-09-13T15:11:19.419029+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:11.472640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.176982+0000","last_change":"2024-09-13T15:10:15.898943+0000","last_active":"2024-09-13T15:11:19.176982+0000","last_peered":"2024-09-13T15:11:19.176982+0000","last_clean":"2024-09-13T15:11:19.176982+0000","last_became_active":"2024-09-13T15:10:15.898631+0000","last_became_peered":"2024-09-13T15:10:15.898631+0000","last_unstale":"2024-09-13T15:11:19.176982+0000","last_undegraded":"2024-09-13T15:11:19.176982+0000","last_fullsized":"2024-09-13T15:11:19.176982+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:18.891010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":120,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418950+0000","last_change":"2024-09-13T15:11:15.161450+0000","last_active":"2024-09-13T15:11:19.418950+0000","last_peered":"2024-09-13T15:11:19.418950+0000","last_clean":"2024-09-13T15:11:19.418950+0000","last_became_active":"2024-09-13T15:11:15.161044+0000","last_became_peered":"2024-09-13T15:11:15.161044+0000","last_unstale":"2024-09-13T15:11:19.418950+0000","last_undegraded":"2024-09-13T15:11:19.418950+0000","last_fullsized":"2024-09-13T15:11:19.418950+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:38.455268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":489,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558369+0000","last_change":"2024-09-13T15:07:52.932503+0000","last_active":"2024-09-13T15:11:19.558369+0000","last_peered":"2024-09-13T15:11:19.558369+0000","last_clean":"2024-09-13T15:11:19.558369+0000","last_became_active":"2024-09-13T15:07:52.932202+0000","last_became_peered":"2024-09-13T15:07:52.932202+0000","last_unstale":"2024-09-13T15:11:19.558369+0000","last_undegraded":"2024-09-13T15:11:19.558369+0000","last_fullsized":"2024-09-13T15:11:19.558369+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:59.204152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":344,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281647+0000","last_change":"2024-09-13T15:10:56.021226+0000","last_active":"2024-09-13T15:11:19.281647+0000","last_peered":"2024-09-13T15:11:19.281647+0000","last_clean":"2024-09-13T15:11:19.281647+0000","last_became_active":"2024-09-13T15:10:56.020925+0000","last_became_peered":"2024-09-13T15:10:56.020925+0000","last_unstale":"2024-09-13T15:11:19.281647+0000","last_undegraded":"2024-09-13T15:11:19.281647+0000","last_fullsized":"2024-09-13T15:11:19.281647+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:33.161549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190384+0000","last_change":"2024-09-13T15:10:04.881950+0000","last_active":"2024-09-13T15:11:24.190384+0000","last_peered":"2024-09-13T15:11:24.190384+0000","last_clean":"2024-09-13T15:11:24.190384+0000","last_became_active":"2024-09-13T15:10:04.881369+0000","last_became_peered":"2024-09-13T15:10:04.881369+0000","last_unstale":"2024-09-13T15:11:24.190384+0000","last_undegraded":"2024-09-13T15:11:24.190384+0000","last_fullsized":"2024-09-13T15:11:24.190384+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:58.868023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":504,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418505+0000","last_change":"2024-09-13T15:11:15.157225+0000","last_active":"2024-09-13T15:11:19.418505+0000","last_peered":"2024-09-13T15:11:19.418505+0000","last_clean":"2024-09-13T15:11:19.418505+0000","last_became_active":"2024-09-13T15:11:15.156885+0000","last_became_peered":"2024-09-13T15:11:15.156885+0000","last_unstale":"2024-09-13T15:11:19.418505+0000","last_undegraded":"2024-09-13T15:11:19.418505+0000","last_fullsized":"2024-09-13T15:11:19.418505+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:24.431933+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":913,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303777+0000","last_change":"2024-09-13T15:10:04.892801+0000","last_active":"2024-09-13T15:11:25.303777+0000","last_peered":"2024-09-13T15:11:25.303777+0000","last_clean":"2024-09-13T15:11:25.303777+0000","last_became_active":"2024-09-13T15:10:04.878965+0000","last_became_peered":"2024-09-13T15:10:04.878965+0000","last_unstale":"2024-09-13T15:11:25.303777+0000","last_undegraded":"2024-09-13T15:11:25.303777+0000","last_fullsized":"2024-09-13T15:11:25.303777+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:58.745084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":183,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177138+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:11:19.177138+0000","last_peered":"2024-09-13T15:11:19.177138+0000","last_clean":"2024-09-13T15:11:19.177138+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:11:19.177138+0000","last_undegraded":"2024-09-13T15:11:19.177138+0000","last_fullsized":"2024-09-13T15:11:19.177138+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":613,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281116+0000","last_change":"2024-09-13T15:10:56.019979+0000","last_active":"2024-09-13T15:11:19.281116+0000","last_peered":"2024-09-13T15:11:19.281116+0000","last_clean":"2024-09-13T15:11:19.281116+0000","last_became_active":"2024-09-13T15:10:56.018652+0000","last_became_peered":"2024-09-13T15:10:56.018652+0000","last_unstale":"2024-09-13T15:11:19.281116+0000","last_undegraded":"2024-09-13T15:11:19.281116+0000","last_fullsized":"2024-09-13T15:11:19.281116+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:42.265583+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:09:42.265583+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:35.758712+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":537,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558480+0000","last_change":"2024-09-13T15:10:15.900103+0000","last_active":"2024-09-13T15:11:19.558480+0000","last_peered":"2024-09-13T15:11:19.558480+0000","last_clean":"2024-09-13T15:11:19.558480+0000","last_became_active":"2024-09-13T15:10:15.898923+0000","last_became_peered":"2024-09-13T15:10:15.898923+0000","last_unstale":"2024-09-13T15:11:19.558480+0000","last_undegraded":"2024-09-13T15:11:19.558480+0000","last_fullsized":"2024-09-13T15:11:19.558480+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:02.884945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177256+0000","last_change":"2024-09-13T15:09:53.831503+0000","last_active":"2024-09-13T15:11:19.177256+0000","last_peered":"2024-09-13T15:11:19.177256+0000","last_clean":"2024-09-13T15:11:19.177256+0000","last_became_active":"2024-09-13T15:09:53.831109+0000","last_became_peered":"2024-09-13T15:09:53.831109+0000","last_unstale":"2024-09-13T15:11:19.177256+0000","last_undegraded":"2024-09-13T15:11:19.177256+0000","last_fullsized":"2024-09-13T15:11:19.177256+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:16:41.523963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.176864+0000","last_change":"2024-09-13T15:09:53.833657+0000","last_active":"2024-09-13T15:11:19.176864+0000","last_peered":"2024-09-13T15:11:19.176864+0000","last_clean":"2024-09-13T15:11:19.176864+0000","last_became_active":"2024-09-13T15:09:53.833270+0000","last_became_peered":"2024-09-13T15:09:53.833270+0000","last_unstale":"2024-09-13T15:11:19.176864+0000","last_undegraded":"2024-09-13T15:11:19.176864+0000","last_fullsized":"2024-09-13T15:11:19.176864+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:34.971883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177492+0000","last_change":"2024-09-13T15:10:15.900211+0000","last_active":"2024-09-13T15:11:19.177492+0000","last_peered":"2024-09-13T15:11:19.177492+0000","last_clean":"2024-09-13T15:11:19.177492+0000","last_became_active":"2024-09-13T15:10:15.899811+0000","last_became_peered":"2024-09-13T15:10:15.899811+0000","last_unstale":"2024-09-13T15:11:19.177492+0000","last_undegraded":"2024-09-13T15:11:19.177492+0000","last_fullsized":"2024-09-13T15:11:19.177492+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:18.102530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":613,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281254+0000","last_change":"2024-09-13T15:10:56.021400+0000","last_active":"2024-09-13T15:11:19.281254+0000","last_peered":"2024-09-13T15:11:19.281254+0000","last_clean":"2024-09-13T15:11:19.281254+0000","last_became_active":"2024-09-13T15:10:56.021067+0000","last_became_peered":"2024-09-13T15:10:56.021067+0000","last_unstale":"2024-09-13T15:11:19.281254+0000","last_undegraded":"2024-09-13T15:11:19.281254+0000","last_fullsized":"2024-09-13T15:11:19.281254+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:57.442552+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:09:57.442552+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:49:08.584114+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":847,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304136+0000","last_change":"2024-09-13T15:10:15.899365+0000","last_active":"2024-09-13T15:11:25.304136+0000","last_peered":"2024-09-13T15:11:25.304136+0000","last_clean":"2024-09-13T15:11:25.304136+0000","last_became_active":"2024-09-13T15:10:15.898995+0000","last_became_peered":"2024-09-13T15:10:15.898995+0000","last_unstale":"2024-09-13T15:11:25.304136+0000","last_undegraded":"2024-09-13T15:11:25.304136+0000","last_fullsized":"2024-09-13T15:11:25.304136+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:36.712427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190142+0000","last_change":"2024-09-13T15:07:52.946957+0000","last_active":"2024-09-13T15:11:24.190142+0000","last_peered":"2024-09-13T15:11:24.190142+0000","last_clean":"2024-09-13T15:11:24.190142+0000","last_became_active":"2024-09-13T15:07:52.946389+0000","last_became_peered":"2024-09-13T15:07:52.946389+0000","last_unstale":"2024-09-13T15:11:24.190142+0000","last_undegraded":"2024-09-13T15:11:24.190142+0000","last_fullsized":"2024-09-13T15:11:24.190142+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.963362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":488,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418530+0000","last_change":"2024-09-13T15:11:15.161189+0000","last_active":"2024-09-13T15:11:19.418530+0000","last_peered":"2024-09-13T15:11:19.418530+0000","last_clean":"2024-09-13T15:11:19.418530+0000","last_became_active":"2024-09-13T15:11:15.160829+0000","last_became_peered":"2024-09-13T15:11:15.160829+0000","last_unstale":"2024-09-13T15:11:19.418530+0000","last_undegraded":"2024-09-13T15:11:19.418530+0000","last_fullsized":"2024-09-13T15:11:19.418530+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:04.813540+0000","scrub_duration":0.0014951610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":359,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281418+0000","last_change":"2024-09-13T15:10:56.024902+0000","last_active":"2024-09-13T15:11:19.281418+0000","last_peered":"2024-09-13T15:11:19.281418+0000","last_clean":"2024-09-13T15:11:19.281418+0000","last_became_active":"2024-09-13T15:10:56.023455+0000","last_became_peered":"2024-09-13T15:10:56.023455+0000","last_unstale":"2024-09-13T15:11:19.281418+0000","last_undegraded":"2024-09-13T15:11:19.281418+0000","last_fullsized":"2024-09-13T15:11:19.281418+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:47.209241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":805,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304206+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:11:25.304206+0000","last_peered":"2024-09-13T15:11:25.304206+0000","last_clean":"2024-09-13T15:11:25.304206+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:11:25.304206+0000","last_undegraded":"2024-09-13T15:11:25.304206+0000","last_fullsized":"2024-09-13T15:11:25.304206+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":481,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558638+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:11:19.558638+0000","last_peered":"2024-09-13T15:11:19.558638+0000","last_clean":"2024-09-13T15:11:19.558638+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:11:19.558638+0000","last_undegraded":"2024-09-13T15:11:19.558638+0000","last_fullsized":"2024-09-13T15:11:19.558638+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":897,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190981+0000","last_change":"2024-09-13T15:07:55.932598+0000","last_active":"2024-09-13T15:11:24.190981+0000","last_peered":"2024-09-13T15:11:24.190981+0000","last_clean":"2024-09-13T15:11:24.190981+0000","last_became_active":"2024-09-13T15:07:55.932277+0000","last_became_peered":"2024-09-13T15:07:55.932277+0000","last_unstale":"2024-09-13T15:11:24.190981+0000","last_undegraded":"2024-09-13T15:11:24.190981+0000","last_fullsized":"2024-09-13T15:11:24.190981+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:56.613460+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":586,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418510+0000","last_change":"2024-09-13T15:11:15.160593+0000","last_active":"2024-09-13T15:11:19.418510+0000","last_peered":"2024-09-13T15:11:19.418510+0000","last_clean":"2024-09-13T15:11:19.418510+0000","last_became_active":"2024-09-13T15:11:15.160234+0000","last_became_peered":"2024-09-13T15:11:15.160234+0000","last_unstale":"2024-09-13T15:11:19.418510+0000","last_undegraded":"2024-09-13T15:11:19.418510+0000","last_fullsized":"2024-09-13T15:11:19.418510+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:34.038625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.189994+0000","last_change":"2024-09-13T15:10:04.877622+0000","last_active":"2024-09-13T15:11:24.189994+0000","last_peered":"2024-09-13T15:11:24.189994+0000","last_clean":"2024-09-13T15:11:24.189994+0000","last_became_active":"2024-09-13T15:10:04.877252+0000","last_became_peered":"2024-09-13T15:10:04.877252+0000","last_unstale":"2024-09-13T15:11:24.189994+0000","last_undegraded":"2024-09-13T15:11:24.189994+0000","last_fullsized":"2024-09-13T15:11:24.189994+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:45.678830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190527+0000","last_change":"2024-09-13T15:10:26.640729+0000","last_active":"2024-09-13T15:11:24.190527+0000","last_peered":"2024-09-13T15:11:24.190527+0000","last_clean":"2024-09-13T15:11:24.190527+0000","last_became_active":"2024-09-13T15:10:26.640399+0000","last_became_peered":"2024-09-13T15:10:26.640399+0000","last_unstale":"2024-09-13T15:11:24.190527+0000","last_undegraded":"2024-09-13T15:11:24.190527+0000","last_fullsized":"2024-09-13T15:11:24.190527+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:22.838751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":504,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418627+0000","last_change":"2024-09-13T15:11:15.162539+0000","last_active":"2024-09-13T15:11:19.418627+0000","last_peered":"2024-09-13T15:11:19.418627+0000","last_clean":"2024-09-13T15:11:19.418627+0000","last_became_active":"2024-09-13T15:11:15.161972+0000","last_became_peered":"2024-09-13T15:11:15.161972+0000","last_unstale":"2024-09-13T15:11:19.418627+0000","last_undegraded":"2024-09-13T15:11:19.418627+0000","last_fullsized":"2024-09-13T15:11:19.418627+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:56.417880+0000","scrub_duration":0.0054109529999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":76,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191660+0000","last_change":"2024-09-13T15:10:04.865457+0000","last_active":"2024-09-13T15:11:24.191660+0000","last_peered":"2024-09-13T15:11:24.191660+0000","last_clean":"2024-09-13T15:11:24.191660+0000","last_became_active":"2024-09-13T15:10:04.865070+0000","last_became_peered":"2024-09-13T15:10:04.865070+0000","last_unstale":"2024-09-13T15:11:24.191660+0000","last_undegraded":"2024-09-13T15:11:24.191660+0000","last_fullsized":"2024-09-13T15:11:24.191660+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:01.050688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190724+0000","last_change":"2024-09-13T15:09:53.824061+0000","last_active":"2024-09-13T15:11:24.190724+0000","last_peered":"2024-09-13T15:11:24.190724+0000","last_clean":"2024-09-13T15:11:24.190724+0000","last_became_active":"2024-09-13T15:09:53.823713+0000","last_became_peered":"2024-09-13T15:09:53.823713+0000","last_unstale":"2024-09-13T15:11:24.190724+0000","last_undegraded":"2024-09-13T15:11:24.190724+0000","last_fullsized":"2024-09-13T15:11:24.190724+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:01.876205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190028+0000","last_change":"2024-09-13T15:10:15.911210+0000","last_active":"2024-09-13T15:11:24.190028+0000","last_peered":"2024-09-13T15:11:24.190028+0000","last_clean":"2024-09-13T15:11:24.190028+0000","last_became_active":"2024-09-13T15:10:15.910792+0000","last_became_peered":"2024-09-13T15:10:15.910792+0000","last_unstale":"2024-09-13T15:11:24.190028+0000","last_undegraded":"2024-09-13T15:11:24.190028+0000","last_fullsized":"2024-09-13T15:11:24.190028+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:53.919718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":662,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281189+0000","last_change":"2024-09-13T15:10:56.018316+0000","last_active":"2024-09-13T15:11:19.281189+0000","last_peered":"2024-09-13T15:11:19.281189+0000","last_clean":"2024-09-13T15:11:19.281189+0000","last_became_active":"2024-09-13T15:10:56.017994+0000","last_became_peered":"2024-09-13T15:10:56.017994+0000","last_unstale":"2024-09-13T15:11:19.281189+0000","last_undegraded":"2024-09-13T15:11:19.281189+0000","last_fullsized":"2024-09-13T15:11:19.281189+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:58.408698+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:09:58.408698+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:03:41.281936+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":360,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281498+0000","last_change":"2024-09-13T15:10:56.021121+0000","last_active":"2024-09-13T15:11:19.281498+0000","last_peered":"2024-09-13T15:11:19.281498+0000","last_clean":"2024-09-13T15:11:19.281498+0000","last_became_active":"2024-09-13T15:10:56.020798+0000","last_became_peered":"2024-09-13T15:10:56.020798+0000","last_unstale":"2024-09-13T15:11:19.281498+0000","last_undegraded":"2024-09-13T15:11:19.281498+0000","last_fullsized":"2024-09-13T15:11:19.281498+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:53:13.477499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210653+0000","last_change":"2024-09-13T15:10:04.877070+0000","last_active":"2024-09-13T15:11:24.210653+0000","last_peered":"2024-09-13T15:11:24.210653+0000","last_clean":"2024-09-13T15:11:24.210653+0000","last_became_active":"2024-09-13T15:10:04.876749+0000","last_became_peered":"2024-09-13T15:10:04.876749+0000","last_unstale":"2024-09-13T15:11:24.210653+0000","last_undegraded":"2024-09-13T15:11:24.210653+0000","last_fullsized":"2024-09-13T15:11:24.210653+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:48.861437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190631+0000","last_change":"2024-09-13T15:10:15.894882+0000","last_active":"2024-09-13T15:11:24.190631+0000","last_peered":"2024-09-13T15:11:24.190631+0000","last_clean":"2024-09-13T15:11:24.190631+0000","last_became_active":"2024-09-13T15:10:15.894220+0000","last_became_peered":"2024-09-13T15:10:15.894220+0000","last_unstale":"2024-09-13T15:11:24.190631+0000","last_undegraded":"2024-09-13T15:11:24.190631+0000","last_fullsized":"2024-09-13T15:11:24.190631+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:59.115938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":748,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558624+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:11:19.558624+0000","last_peered":"2024-09-13T15:11:19.558624+0000","last_clean":"2024-09-13T15:11:19.558624+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:11:19.558624+0000","last_undegraded":"2024-09-13T15:11:19.558624+0000","last_fullsized":"2024-09-13T15:11:19.558624+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":363,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281597+0000","last_change":"2024-09-13T15:10:56.019310+0000","last_active":"2024-09-13T15:11:19.281597+0000","last_peered":"2024-09-13T15:11:19.281597+0000","last_clean":"2024-09-13T15:11:19.281597+0000","last_became_active":"2024-09-13T15:10:56.019013+0000","last_became_peered":"2024-09-13T15:10:56.019013+0000","last_unstale":"2024-09-13T15:11:19.281597+0000","last_undegraded":"2024-09-13T15:11:19.281597+0000","last_fullsized":"2024-09-13T15:11:19.281597+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:29.063329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":175,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190846+0000","last_change":"2024-09-13T15:10:04.869226+0000","last_active":"2024-09-13T15:11:24.190846+0000","last_peered":"2024-09-13T15:11:24.190846+0000","last_clean":"2024-09-13T15:11:24.190846+0000","last_became_active":"2024-09-13T15:10:04.868804+0000","last_became_peered":"2024-09-13T15:10:04.868804+0000","last_unstale":"2024-09-13T15:11:24.190846+0000","last_undegraded":"2024-09-13T15:11:24.190846+0000","last_fullsized":"2024-09-13T15:11:24.190846+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:46.742018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190130+0000","last_change":"2024-09-13T15:10:15.911230+0000","last_active":"2024-09-13T15:11:24.190130+0000","last_peered":"2024-09-13T15:11:24.190130+0000","last_clean":"2024-09-13T15:11:24.190130+0000","last_became_active":"2024-09-13T15:10:15.910902+0000","last_became_peered":"2024-09-13T15:10:15.910902+0000","last_unstale":"2024-09-13T15:11:24.190130+0000","last_undegraded":"2024-09-13T15:11:24.190130+0000","last_fullsized":"2024-09-13T15:11:24.190130+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:54.995764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":831,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558631+0000","last_change":"2024-09-13T15:10:07.870238+0000","last_active":"2024-09-13T15:11:19.558631+0000","last_peered":"2024-09-13T15:11:19.558631+0000","last_clean":"2024-09-13T15:11:19.558631+0000","last_became_active":"2024-09-13T15:10:07.869795+0000","last_became_peered":"2024-09-13T15:10:07.869795+0000","last_unstale":"2024-09-13T15:11:19.558631+0000","last_undegraded":"2024-09-13T15:11:19.558631+0000","last_fullsized":"2024-09-13T15:11:19.558631+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:47.671154+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190547+0000","last_change":"2024-09-13T15:10:15.911505+0000","last_active":"2024-09-13T15:11:24.190547+0000","last_peered":"2024-09-13T15:11:24.190547+0000","last_clean":"2024-09-13T15:11:24.190547+0000","last_became_active":"2024-09-13T15:10:15.910951+0000","last_became_peered":"2024-09-13T15:10:15.910951+0000","last_unstale":"2024-09-13T15:11:24.190547+0000","last_undegraded":"2024-09-13T15:11:24.190547+0000","last_fullsized":"2024-09-13T15:11:24.190547+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:41.400873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":256,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418939+0000","last_change":"2024-09-13T15:11:15.163170+0000","last_active":"2024-09-13T15:11:19.418939+0000","last_peered":"2024-09-13T15:11:19.418939+0000","last_clean":"2024-09-13T15:11:19.418939+0000","last_became_active":"2024-09-13T15:11:15.162898+0000","last_became_peered":"2024-09-13T15:11:15.162898+0000","last_unstale":"2024-09-13T15:11:19.418939+0000","last_undegraded":"2024-09-13T15:11:19.418939+0000","last_fullsized":"2024-09-13T15:11:19.418939+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:56.176185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190299+0000","last_change":"2024-09-13T15:10:04.875459+0000","last_active":"2024-09-13T15:11:24.190299+0000","last_peered":"2024-09-13T15:11:24.190299+0000","last_clean":"2024-09-13T15:11:24.190299+0000","last_became_active":"2024-09-13T15:10:04.874915+0000","last_became_peered":"2024-09-13T15:10:04.874915+0000","last_unstale":"2024-09-13T15:11:24.190299+0000","last_undegraded":"2024-09-13T15:11:24.190299+0000","last_fullsized":"2024-09-13T15:11:24.190299+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:04.852075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":711,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210779+0000","last_change":"2024-09-13T15:09:56.647357+0000","last_active":"2024-09-13T15:11:24.210779+0000","last_peered":"2024-09-13T15:11:24.210779+0000","last_clean":"2024-09-13T15:11:24.210779+0000","last_became_active":"2024-09-13T15:09:56.646785+0000","last_became_peered":"2024-09-13T15:09:56.646785+0000","last_unstale":"2024-09-13T15:11:24.210779+0000","last_undegraded":"2024-09-13T15:11:24.210779+0000","last_fullsized":"2024-09-13T15:11:24.210779+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:13.336788+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":860,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303779+0000","last_change":"2024-09-13T15:10:04.888105+0000","last_active":"2024-09-13T15:11:25.303779+0000","last_peered":"2024-09-13T15:11:25.303779+0000","last_clean":"2024-09-13T15:11:25.303779+0000","last_became_active":"2024-09-13T15:10:04.878585+0000","last_became_peered":"2024-09-13T15:10:04.878585+0000","last_unstale":"2024-09-13T15:11:25.303779+0000","last_undegraded":"2024-09-13T15:11:25.303779+0000","last_fullsized":"2024-09-13T15:11:25.303779+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:16.156696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191049+0000","last_change":"2024-09-13T15:10:15.904203+0000","last_active":"2024-09-13T15:11:24.191049+0000","last_peered":"2024-09-13T15:11:24.191049+0000","last_clean":"2024-09-13T15:11:24.191049+0000","last_became_active":"2024-09-13T15:10:15.903806+0000","last_became_peered":"2024-09-13T15:10:15.903806+0000","last_unstale":"2024-09-13T15:11:24.191049+0000","last_undegraded":"2024-09-13T15:11:24.191049+0000","last_fullsized":"2024-09-13T15:11:24.191049+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:35.485847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":870,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303674+0000","last_change":"2024-09-13T15:10:04.878380+0000","last_active":"2024-09-13T15:11:25.303674+0000","last_peered":"2024-09-13T15:11:25.303674+0000","last_clean":"2024-09-13T15:11:25.303674+0000","last_became_active":"2024-09-13T15:10:04.877985+0000","last_became_peered":"2024-09-13T15:10:04.877985+0000","last_unstale":"2024-09-13T15:11:25.303674+0000","last_undegraded":"2024-09-13T15:11:25.303674+0000","last_fullsized":"2024-09-13T15:11:25.303674+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:43.195981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":1078,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304341+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:11:25.304341+0000","last_peered":"2024-09-13T15:11:25.304341+0000","last_clean":"2024-09-13T15:11:25.304341+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:11:25.304341+0000","last_undegraded":"2024-09-13T15:11:25.304341+0000","last_fullsized":"2024-09-13T15:11:25.304341+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":73,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190701+0000","last_change":"2024-09-13T15:10:15.905192+0000","last_active":"2024-09-13T15:11:24.190701+0000","last_peered":"2024-09-13T15:11:24.190701+0000","last_clean":"2024-09-13T15:11:24.190701+0000","last_became_active":"2024-09-13T15:10:15.899075+0000","last_became_peered":"2024-09-13T15:10:15.899075+0000","last_unstale":"2024-09-13T15:11:24.190701+0000","last_undegraded":"2024-09-13T15:11:24.190701+0000","last_fullsized":"2024-09-13T15:11:24.190701+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:28.249717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":503,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.047821+0000","last_change":"2024-09-13T15:09:53.827985+0000","last_active":"2024-09-13T15:11:25.047821+0000","last_peered":"2024-09-13T15:11:25.047821+0000","last_clean":"2024-09-13T15:11:25.047821+0000","last_became_active":"2024-09-13T15:09:53.827316+0000","last_became_peered":"2024-09-13T15:09:53.827316+0000","last_unstale":"2024-09-13T15:11:25.047821+0000","last_undegraded":"2024-09-13T15:11:25.047821+0000","last_fullsized":"2024-09-13T15:11:25.047821+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:01.122031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":184,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210217+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:11:24.210217+0000","last_peered":"2024-09-13T15:11:24.210217+0000","last_clean":"2024-09-13T15:11:24.210217+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:11:24.210217+0000","last_undegraded":"2024-09-13T15:11:24.210217+0000","last_fullsized":"2024-09-13T15:11:24.210217+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177312+0000","last_change":"2024-09-13T15:10:15.900038+0000","last_active":"2024-09-13T15:11:19.177312+0000","last_peered":"2024-09-13T15:11:19.177312+0000","last_clean":"2024-09-13T15:11:19.177312+0000","last_became_active":"2024-09-13T15:10:15.899620+0000","last_became_peered":"2024-09-13T15:10:15.899620+0000","last_unstale":"2024-09-13T15:11:19.177312+0000","last_undegraded":"2024-09-13T15:11:19.177312+0000","last_fullsized":"2024-09-13T15:11:19.177312+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:20.496234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":193,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177139+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:11:19.177139+0000","last_peered":"2024-09-13T15:11:19.177139+0000","last_clean":"2024-09-13T15:11:19.177139+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:11:19.177139+0000","last_undegraded":"2024-09-13T15:11:19.177139+0000","last_fullsized":"2024-09-13T15:11:19.177139+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190267+0000","last_change":"2024-09-13T15:10:15.900173+0000","last_active":"2024-09-13T15:11:24.190267+0000","last_peered":"2024-09-13T15:11:24.190267+0000","last_clean":"2024-09-13T15:11:24.190267+0000","last_became_active":"2024-09-13T15:10:15.899834+0000","last_became_peered":"2024-09-13T15:10:15.899834+0000","last_unstale":"2024-09-13T15:11:24.190267+0000","last_undegraded":"2024-09-13T15:11:24.190267+0000","last_fullsized":"2024-09-13T15:11:24.190267+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:06.566755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":29,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418828+0000","last_change":"2024-09-13T15:11:15.162666+0000","last_active":"2024-09-13T15:11:19.418828+0000","last_peered":"2024-09-13T15:11:19.418828+0000","last_clean":"2024-09-13T15:11:19.418828+0000","last_became_active":"2024-09-13T15:11:15.162307+0000","last_became_peered":"2024-09-13T15:11:15.162307+0000","last_unstale":"2024-09-13T15:11:19.418828+0000","last_undegraded":"2024-09-13T15:11:19.418828+0000","last_fullsized":"2024-09-13T15:11:19.418828+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:40.717528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":342,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281538+0000","last_change":"2024-09-13T15:10:56.022799+0000","last_active":"2024-09-13T15:11:19.281538+0000","last_peered":"2024-09-13T15:11:19.281538+0000","last_clean":"2024-09-13T15:11:19.281538+0000","last_became_active":"2024-09-13T15:10:56.022484+0000","last_became_peered":"2024-09-13T15:10:56.022484+0000","last_unstale":"2024-09-13T15:11:19.281538+0000","last_undegraded":"2024-09-13T15:11:19.281538+0000","last_fullsized":"2024-09-13T15:11:19.281538+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:12.202093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281200+0000","last_change":"2024-09-13T15:10:56.023490+0000","last_active":"2024-09-13T15:11:19.281200+0000","last_peered":"2024-09-13T15:11:19.281200+0000","last_clean":"2024-09-13T15:11:19.281200+0000","last_became_active":"2024-09-13T15:10:56.022883+0000","last_became_peered":"2024-09-13T15:10:56.022883+0000","last_unstale":"2024-09-13T15:11:19.281200+0000","last_undegraded":"2024-09-13T15:11:19.281200+0000","last_fullsized":"2024-09-13T15:11:19.281200+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:22.402212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190201+0000","last_change":"2024-09-13T15:10:04.876037+0000","last_active":"2024-09-13T15:11:24.190201+0000","last_peered":"2024-09-13T15:11:24.190201+0000","last_clean":"2024-09-13T15:11:24.190201+0000","last_became_active":"2024-09-13T15:10:04.875624+0000","last_became_peered":"2024-09-13T15:10:04.875624+0000","last_unstale":"2024-09-13T15:11:24.190201+0000","last_undegraded":"2024-09-13T15:11:24.190201+0000","last_fullsized":"2024-09-13T15:11:24.190201+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:12.697390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":844,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303956+0000","last_change":"2024-09-13T15:10:15.898047+0000","last_active":"2024-09-13T15:11:25.303956+0000","last_peered":"2024-09-13T15:11:25.303956+0000","last_clean":"2024-09-13T15:11:25.303956+0000","last_became_active":"2024-09-13T15:10:15.897466+0000","last_became_peered":"2024-09-13T15:10:15.897466+0000","last_unstale":"2024-09-13T15:11:25.303956+0000","last_undegraded":"2024-09-13T15:11:25.303956+0000","last_fullsized":"2024-09-13T15:11:25.303956+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:03.737634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":490,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558694+0000","last_change":"2024-09-13T15:07:52.936799+0000","last_active":"2024-09-13T15:11:19.558694+0000","last_peered":"2024-09-13T15:11:19.558694+0000","last_clean":"2024-09-13T15:11:19.558694+0000","last_became_active":"2024-09-13T15:07:52.936436+0000","last_became_peered":"2024-09-13T15:07:52.936436+0000","last_unstale":"2024-09-13T15:11:19.558694+0000","last_undegraded":"2024-09-13T15:11:19.558694+0000","last_fullsized":"2024-09-13T15:11:19.558694+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:52:09.480999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":913,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304108+0000","last_change":"2024-09-13T15:10:15.894390+0000","last_active":"2024-09-13T15:11:25.304108+0000","last_peered":"2024-09-13T15:11:25.304108+0000","last_clean":"2024-09-13T15:11:25.304108+0000","last_became_active":"2024-09-13T15:10:15.894013+0000","last_became_peered":"2024-09-13T15:10:15.894013+0000","last_unstale":"2024-09-13T15:11:25.304108+0000","last_undegraded":"2024-09-13T15:11:25.304108+0000","last_fullsized":"2024-09-13T15:11:25.304108+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:15.836830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190833+0000","last_change":"2024-09-13T15:10:04.881987+0000","last_active":"2024-09-13T15:11:24.190833+0000","last_peered":"2024-09-13T15:11:24.190833+0000","last_clean":"2024-09-13T15:11:24.190833+0000","last_became_active":"2024-09-13T15:10:04.881601+0000","last_became_peered":"2024-09-13T15:10:04.881601+0000","last_unstale":"2024-09-13T15:11:24.190833+0000","last_undegraded":"2024-09-13T15:11:24.190833+0000","last_fullsized":"2024-09-13T15:11:24.190833+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:39.286013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":153,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.189994+0000","last_change":"2024-09-13T15:07:52.936887+0000","last_active":"2024-09-13T15:11:24.189994+0000","last_peered":"2024-09-13T15:11:24.189994+0000","last_clean":"2024-09-13T15:11:24.189994+0000","last_became_active":"2024-09-13T15:07:52.936398+0000","last_became_peered":"2024-09-13T15:07:52.936398+0000","last_unstale":"2024-09-13T15:11:24.189994+0000","last_undegraded":"2024-09-13T15:11:24.189994+0000","last_fullsized":"2024-09-13T15:11:24.189994+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:05.892375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":843,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304298+0000","last_change":"2024-09-13T15:07:52.935989+0000","last_active":"2024-09-13T15:11:25.304298+0000","last_peered":"2024-09-13T15:11:25.304298+0000","last_clean":"2024-09-13T15:11:25.304298+0000","last_became_active":"2024-09-13T15:07:52.935661+0000","last_became_peered":"2024-09-13T15:07:52.935661+0000","last_unstale":"2024-09-13T15:11:25.304298+0000","last_undegraded":"2024-09-13T15:11:25.304298+0000","last_fullsized":"2024-09-13T15:11:25.304298+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:49.027141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":219,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210373+0000","last_change":"2024-09-13T15:09:53.829469+0000","last_active":"2024-09-13T15:11:24.210373+0000","last_peered":"2024-09-13T15:11:24.210373+0000","last_clean":"2024-09-13T15:11:24.210373+0000","last_became_active":"2024-09-13T15:09:53.829050+0000","last_became_peered":"2024-09-13T15:09:53.829050+0000","last_unstale":"2024-09-13T15:11:24.210373+0000","last_undegraded":"2024-09-13T15:11:24.210373+0000","last_fullsized":"2024-09-13T15:11:24.210373+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:54.177468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191781+0000","last_change":"2024-09-13T15:09:53.829973+0000","last_active":"2024-09-13T15:11:24.191781+0000","last_peered":"2024-09-13T15:11:24.191781+0000","last_clean":"2024-09-13T15:11:24.191781+0000","last_became_active":"2024-09-13T15:09:53.829601+0000","last_became_peered":"2024-09-13T15:09:53.829601+0000","last_unstale":"2024-09-13T15:11:24.191781+0000","last_undegraded":"2024-09-13T15:11:24.191781+0000","last_fullsized":"2024-09-13T15:11:24.191781+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:47.834948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190201+0000","last_change":"2024-09-13T15:10:04.882777+0000","last_active":"2024-09-13T15:11:24.190201+0000","last_peered":"2024-09-13T15:11:24.190201+0000","last_clean":"2024-09-13T15:11:24.190201+0000","last_became_active":"2024-09-13T15:10:04.882416+0000","last_became_peered":"2024-09-13T15:10:04.882416+0000","last_unstale":"2024-09-13T15:11:24.190201+0000","last_undegraded":"2024-09-13T15:11:24.190201+0000","last_fullsized":"2024-09-13T15:11:24.190201+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:34.906936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":228,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210537+0000","last_change":"2024-09-13T15:10:15.899883+0000","last_active":"2024-09-13T15:11:24.210537+0000","last_peered":"2024-09-13T15:11:24.210537+0000","last_clean":"2024-09-13T15:11:24.210537+0000","last_became_active":"2024-09-13T15:10:15.899431+0000","last_became_peered":"2024-09-13T15:10:15.899431+0000","last_unstale":"2024-09-13T15:11:24.210537+0000","last_undegraded":"2024-09-13T15:11:24.210537+0000","last_fullsized":"2024-09-13T15:11:24.210537+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:37.343699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":370,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281632+0000","last_change":"2024-09-13T15:10:56.021939+0000","last_active":"2024-09-13T15:11:19.281632+0000","last_peered":"2024-09-13T15:11:19.281632+0000","last_clean":"2024-09-13T15:11:19.281632+0000","last_became_active":"2024-09-13T15:10:56.021547+0000","last_became_peered":"2024-09-13T15:10:56.021547+0000","last_unstale":"2024-09-13T15:11:19.281632+0000","last_undegraded":"2024-09-13T15:11:19.281632+0000","last_fullsized":"2024-09-13T15:11:19.281632+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:49:50.175522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":230,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210303+0000","last_change":"2024-09-13T15:10:04.867225+0000","last_active":"2024-09-13T15:11:24.210303+0000","last_peered":"2024-09-13T15:11:24.210303+0000","last_clean":"2024-09-13T15:11:24.210303+0000","last_became_active":"2024-09-13T15:10:04.866809+0000","last_became_peered":"2024-09-13T15:10:04.866809+0000","last_unstale":"2024-09-13T15:11:24.210303+0000","last_undegraded":"2024-09-13T15:11:24.210303+0000","last_fullsized":"2024-09-13T15:11:24.210303+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:32.561235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":355,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281623+0000","last_change":"2024-09-13T15:10:56.023584+0000","last_active":"2024-09-13T15:11:19.281623+0000","last_peered":"2024-09-13T15:11:19.281623+0000","last_clean":"2024-09-13T15:11:19.281623+0000","last_became_active":"2024-09-13T15:10:56.022873+0000","last_became_peered":"2024-09-13T15:10:56.022873+0000","last_unstale":"2024-09-13T15:11:19.281623+0000","last_undegraded":"2024-09-13T15:11:19.281623+0000","last_fullsized":"2024-09-13T15:11:19.281623+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:56:16.232555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":901,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303906+0000","last_change":"2024-09-13T15:10:04.877029+0000","last_active":"2024-09-13T15:11:25.303906+0000","last_peered":"2024-09-13T15:11:25.303906+0000","last_clean":"2024-09-13T15:11:25.303906+0000","last_became_active":"2024-09-13T15:10:04.876676+0000","last_became_peered":"2024-09-13T15:10:04.876676+0000","last_unstale":"2024-09-13T15:11:25.303906+0000","last_undegraded":"2024-09-13T15:11:25.303906+0000","last_fullsized":"2024-09-13T15:11:25.303906+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:16.624424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":853,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304511+0000","last_change":"2024-09-13T15:09:53.833193+0000","last_active":"2024-09-13T15:11:25.304511+0000","last_peered":"2024-09-13T15:11:25.304511+0000","last_clean":"2024-09-13T15:11:25.304511+0000","last_became_active":"2024-09-13T15:09:53.832777+0000","last_became_peered":"2024-09-13T15:09:53.832777+0000","last_unstale":"2024-09-13T15:11:25.304511+0000","last_undegraded":"2024-09-13T15:11:25.304511+0000","last_fullsized":"2024-09-13T15:11:25.304511+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:13:59.972608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:21.922268+0000","last_change":"2024-09-13T15:07:52.933733+0000","last_active":"2024-09-13T15:11:21.922268+0000","last_peered":"2024-09-13T15:11:21.922268+0000","last_clean":"2024-09-13T15:11:21.922268+0000","last_became_active":"2024-09-13T15:07:52.933425+0000","last_became_peered":"2024-09-13T15:07:52.933425+0000","last_unstale":"2024-09-13T15:11:21.922268+0000","last_undegraded":"2024-09-13T15:11:21.922268+0000","last_fullsized":"2024-09-13T15:11:21.922268+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:59.564486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":64,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190420+0000","last_change":"2024-09-13T15:10:04.887555+0000","last_active":"2024-09-13T15:11:24.190420+0000","last_peered":"2024-09-13T15:11:24.190420+0000","last_clean":"2024-09-13T15:11:24.190420+0000","last_became_active":"2024-09-13T15:10:04.873030+0000","last_became_peered":"2024-09-13T15:10:04.873030+0000","last_unstale":"2024-09-13T15:11:24.190420+0000","last_undegraded":"2024-09-13T15:11:24.190420+0000","last_fullsized":"2024-09-13T15:11:24.190420+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:14.195107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":5954,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234625,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429772,"kb_used_data":6656,"kb_used_omap":1,"kb_used_meta":1423102,"kb_avail":92315380,"statfs":{"total":95995035648,"available":94530949120,"internally_reserved":0,"allocated":6815744,"data_stored":4232922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1457256907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496263,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455620,"kb_used_data":6328,"kb_used_omap":1,"kb_used_meta":1449278,"kb_avail":92289532,"statfs":{"total":95995035648,"available":94504480768,"internally_reserved":0,"allocated":6479872,"data_stored":3905115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1484061127},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757901,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700796,"kb_used_data":6896,"kb_used_omap":1,"kb_used_meta":1693886,"kb_avail":92044356,"statfs":{"total":95995035648,"available":94253420544,"internally_reserved":0,"allocated":7061504,"data_stored":4490978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1734539718},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019537,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351560,"kb_used_data":6332,"kb_used_omap":1,"kb_used_meta":1345214,"kb_avail":92393592,"statfs":{"total":95995035648,"available":94611038208,"internally_reserved":0,"allocated":6483968,"data_stored":3905172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1377499590},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650750,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1156536,"kb_used_data":6892,"kb_used_omap":4,"kb_used_meta":1149627,"kb_avail":92588616,"statfs":{"total":95995035648,"available":94810742784,"internally_reserved":0,"allocated":7057408,"data_stored":4490740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":1177218607},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879685,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145276,"kb_used_data":6576,"kb_used_omap":6,"kb_used_meta":1138681,"kb_avail":92599876,"statfs":{"total":95995035648,"available":94822273024,"internally_reserved":0,"allocated":6733824,"data_stored":4163586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":1166009913},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435701,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321964,"kb_used_data":6304,"kb_used_omap":5,"kb_used_meta":1315642,"kb_avail":92423188,"statfs":{"total":95995035648,"available":94641344512,"internally_reserved":0,"allocated":6455296,"data_stored":3899755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1347217761},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":348,"seq":1494648619144,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":114104,"kb_used_data":6316,"kb_used_omap":10,"kb_used_meta":107765,"kb_avail":93631048,"statfs":{"total":95995035648,"available":95878193152,"internally_reserved":0,"allocated":6467584,"data_stored":3900886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10675,"internal_metadata":110351949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:12:13.320 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T15:12:14.119 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T15:12:14.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[113926]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[113926]: pgmap v1727: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[113926]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[108570]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[108570]: pgmap v1727: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:14.599 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:14 smithi042 ceph-mon[108570]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:14 smithi159 ceph-mon[102519]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T15:12:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:14 smithi159 ceph-mon[102519]: pgmap v1727: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:14.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:14 smithi159 ceph-mon[102519]: from='mgr.34107 172.21.15.42:0/3686295077' entity='mgr.y' 2024-09-13T15:12:16.513 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:16 smithi042 ceph-mon[108570]: pgmap v1728: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:16.513 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:16 smithi042 ceph-mon[113926]: pgmap v1728: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:16.707 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T15:12:16.707 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T15:12:16.749 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:16 smithi159 ceph-mon[102519]: pgmap v1728: 83 pgs: 83 active+clean; 582 KiB data, 9.2 GiB used, 706 GiB / 715 GiB avail 2024-09-13T15:12:17.428 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":1728,"stamp":"2024-09-13T15:12:14.837429+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":5954,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":249,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9675628,"kb_used_data":52300,"kb_used_omap":33,"kb_used_meta":9623198,"kb_avail":740285588,"statfs":{"total":767960285184,"available":758052442112,"internally_reserved":0,"allocated":53555200,"data_stored":32989154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33996,"internal_metadata":9854155572},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003132"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":851,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304357+0000","last_change":"2024-09-13T15:10:26.313323+0000","last_active":"2024-09-13T15:11:25.304357+0000","last_peered":"2024-09-13T15:11:25.304357+0000","last_clean":"2024-09-13T15:11:25.304357+0000","last_became_active":"2024-09-13T15:10:26.312986+0000","last_became_peered":"2024-09-13T15:10:26.312986+0000","last_unstale":"2024-09-13T15:11:25.304357+0000","last_undegraded":"2024-09-13T15:11:25.304357+0000","last_fullsized":"2024-09-13T15:11:25.304357+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:07.615254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"71'2","reported_seq":150,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418821+0000","last_change":"2024-09-13T15:11:15.162702+0000","last_active":"2024-09-13T15:11:19.418821+0000","last_peered":"2024-09-13T15:11:19.418821+0000","last_clean":"2024-09-13T15:11:19.418821+0000","last_became_active":"2024-09-13T15:11:15.162324+0000","last_became_peered":"2024-09-13T15:11:15.162324+0000","last_unstale":"2024-09-13T15:11:19.418821+0000","last_undegraded":"2024-09-13T15:11:19.418821+0000","last_fullsized":"2024-09-13T15:11:19.418821+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:44:49.486702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":341,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281461+0000","last_change":"2024-09-13T15:10:56.023607+0000","last_active":"2024-09-13T15:11:19.281461+0000","last_peered":"2024-09-13T15:11:19.281461+0000","last_clean":"2024-09-13T15:11:19.281461+0000","last_became_active":"2024-09-13T15:10:56.023279+0000","last_became_peered":"2024-09-13T15:10:56.023279+0000","last_unstale":"2024-09-13T15:11:19.281461+0000","last_undegraded":"2024-09-13T15:11:19.281461+0000","last_fullsized":"2024-09-13T15:11:19.281461+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:50:57.733968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.189994+0000","last_change":"2024-09-13T15:10:04.872212+0000","last_active":"2024-09-13T15:11:24.189994+0000","last_peered":"2024-09-13T15:11:24.189994+0000","last_clean":"2024-09-13T15:11:24.189994+0000","last_became_active":"2024-09-13T15:10:04.871786+0000","last_became_peered":"2024-09-13T15:10:04.871786+0000","last_unstale":"2024-09-13T15:11:24.189994+0000","last_undegraded":"2024-09-13T15:11:24.189994+0000","last_fullsized":"2024-09-13T15:11:24.189994+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:53:50.179272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190612+0000","last_change":"2024-09-13T15:10:04.876599+0000","last_active":"2024-09-13T15:11:24.190612+0000","last_peered":"2024-09-13T15:11:24.190612+0000","last_clean":"2024-09-13T15:11:24.190612+0000","last_became_active":"2024-09-13T15:10:04.876243+0000","last_became_peered":"2024-09-13T15:10:04.876243+0000","last_unstale":"2024-09-13T15:11:24.190612+0000","last_undegraded":"2024-09-13T15:11:24.190612+0000","last_fullsized":"2024-09-13T15:11:24.190612+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:47:40.860759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418671+0000","last_change":"2024-09-13T15:11:15.162927+0000","last_active":"2024-09-13T15:11:19.418671+0000","last_peered":"2024-09-13T15:11:19.418671+0000","last_clean":"2024-09-13T15:11:19.418671+0000","last_became_active":"2024-09-13T15:11:15.162626+0000","last_became_peered":"2024-09-13T15:11:15.162626+0000","last_unstale":"2024-09-13T15:11:19.418671+0000","last_undegraded":"2024-09-13T15:11:19.418671+0000","last_fullsized":"2024-09-13T15:11:19.418671+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:37:36.424513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":856,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304488+0000","last_change":"2024-09-13T15:10:26.313311+0000","last_active":"2024-09-13T15:11:25.304488+0000","last_peered":"2024-09-13T15:11:25.304488+0000","last_clean":"2024-09-13T15:11:25.304488+0000","last_became_active":"2024-09-13T15:10:26.313011+0000","last_became_peered":"2024-09-13T15:10:26.313011+0000","last_unstale":"2024-09-13T15:11:25.304488+0000","last_undegraded":"2024-09-13T15:11:25.304488+0000","last_fullsized":"2024-09-13T15:11:25.304488+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:20:18.847442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"71'8","reported_seq":608,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.047050+0000","last_change":"2024-09-13T15:10:04.883100+0000","last_active":"2024-09-13T15:11:25.047050+0000","last_peered":"2024-09-13T15:11:25.047050+0000","last_clean":"2024-09-13T15:11:25.047050+0000","last_became_active":"2024-09-13T15:10:04.869463+0000","last_became_peered":"2024-09-13T15:10:04.869463+0000","last_unstale":"2024-09-13T15:11:25.047050+0000","last_undegraded":"2024-09-13T15:11:25.047050+0000","last_fullsized":"2024-09-13T15:11:25.047050+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:07:18.719681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"71'1","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.419094+0000","last_change":"2024-09-13T15:11:15.162536+0000","last_active":"2024-09-13T15:11:19.419094+0000","last_peered":"2024-09-13T15:11:19.419094+0000","last_clean":"2024-09-13T15:11:19.419094+0000","last_became_active":"2024-09-13T15:11:15.162161+0000","last_became_peered":"2024-09-13T15:11:15.162161+0000","last_unstale":"2024-09-13T15:11:19.419094+0000","last_undegraded":"2024-09-13T15:11:19.419094+0000","last_fullsized":"2024-09-13T15:11:19.419094+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:01:47.281436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":328,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281211+0000","last_change":"2024-09-13T15:10:56.023909+0000","last_active":"2024-09-13T15:11:19.281211+0000","last_peered":"2024-09-13T15:11:19.281211+0000","last_clean":"2024-09-13T15:11:19.281211+0000","last_became_active":"2024-09-13T15:10:56.023038+0000","last_became_peered":"2024-09-13T15:10:56.023038+0000","last_unstale":"2024-09-13T15:11:19.281211+0000","last_undegraded":"2024-09-13T15:11:19.281211+0000","last_fullsized":"2024-09-13T15:11:19.281211+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:36.486723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"71'2","reported_seq":136,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.419029+0000","last_change":"2024-09-13T15:11:15.162813+0000","last_active":"2024-09-13T15:11:19.419029+0000","last_peered":"2024-09-13T15:11:19.419029+0000","last_clean":"2024-09-13T15:11:19.419029+0000","last_became_active":"2024-09-13T15:11:15.162071+0000","last_became_peered":"2024-09-13T15:11:15.162071+0000","last_unstale":"2024-09-13T15:11:19.419029+0000","last_undegraded":"2024-09-13T15:11:19.419029+0000","last_fullsized":"2024-09-13T15:11:19.419029+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:19:11.472640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.176982+0000","last_change":"2024-09-13T15:10:15.898943+0000","last_active":"2024-09-13T15:11:19.176982+0000","last_peered":"2024-09-13T15:11:19.176982+0000","last_clean":"2024-09-13T15:11:19.176982+0000","last_became_active":"2024-09-13T15:10:15.898631+0000","last_became_peered":"2024-09-13T15:10:15.898631+0000","last_unstale":"2024-09-13T15:11:19.176982+0000","last_undegraded":"2024-09-13T15:11:19.176982+0000","last_fullsized":"2024-09-13T15:11:19.176982+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:20:18.891010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"71'1","reported_seq":120,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418950+0000","last_change":"2024-09-13T15:11:15.161450+0000","last_active":"2024-09-13T15:11:19.418950+0000","last_peered":"2024-09-13T15:11:19.418950+0000","last_clean":"2024-09-13T15:11:19.418950+0000","last_became_active":"2024-09-13T15:11:15.161044+0000","last_became_peered":"2024-09-13T15:11:15.161044+0000","last_unstale":"2024-09-13T15:11:19.418950+0000","last_undegraded":"2024-09-13T15:11:19.418950+0000","last_fullsized":"2024-09-13T15:11:19.418950+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:06:38.455268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":489,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558369+0000","last_change":"2024-09-13T15:07:52.932503+0000","last_active":"2024-09-13T15:11:19.558369+0000","last_peered":"2024-09-13T15:11:19.558369+0000","last_clean":"2024-09-13T15:11:19.558369+0000","last_became_active":"2024-09-13T15:07:52.932202+0000","last_became_peered":"2024-09-13T15:07:52.932202+0000","last_unstale":"2024-09-13T15:11:19.558369+0000","last_undegraded":"2024-09-13T15:11:19.558369+0000","last_fullsized":"2024-09-13T15:11:19.558369+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:51:59.204152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":344,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281647+0000","last_change":"2024-09-13T15:10:56.021226+0000","last_active":"2024-09-13T15:11:19.281647+0000","last_peered":"2024-09-13T15:11:19.281647+0000","last_clean":"2024-09-13T15:11:19.281647+0000","last_became_active":"2024-09-13T15:10:56.020925+0000","last_became_peered":"2024-09-13T15:10:56.020925+0000","last_unstale":"2024-09-13T15:11:19.281647+0000","last_undegraded":"2024-09-13T15:11:19.281647+0000","last_fullsized":"2024-09-13T15:11:19.281647+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:28:33.161549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190384+0000","last_change":"2024-09-13T15:10:04.881950+0000","last_active":"2024-09-13T15:11:24.190384+0000","last_peered":"2024-09-13T15:11:24.190384+0000","last_clean":"2024-09-13T15:11:24.190384+0000","last_became_active":"2024-09-13T15:10:04.881369+0000","last_became_peered":"2024-09-13T15:10:04.881369+0000","last_unstale":"2024-09-13T15:11:24.190384+0000","last_undegraded":"2024-09-13T15:11:24.190384+0000","last_fullsized":"2024-09-13T15:11:24.190384+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:17:58.868023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"85'149","reported_seq":504,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418505+0000","last_change":"2024-09-13T15:11:15.157225+0000","last_active":"2024-09-13T15:11:19.418505+0000","last_peered":"2024-09-13T15:11:19.418505+0000","last_clean":"2024-09-13T15:11:19.418505+0000","last_became_active":"2024-09-13T15:11:15.156885+0000","last_became_peered":"2024-09-13T15:11:15.156885+0000","last_unstale":"2024-09-13T15:11:19.418505+0000","last_undegraded":"2024-09-13T15:11:19.418505+0000","last_fullsized":"2024-09-13T15:11:19.418505+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:51.810715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:51.810715+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:17:24.431933+0000","scrub_duration":0.0039778210000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"71'2","reported_seq":913,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303777+0000","last_change":"2024-09-13T15:10:04.892801+0000","last_active":"2024-09-13T15:11:25.303777+0000","last_peered":"2024-09-13T15:11:25.303777+0000","last_clean":"2024-09-13T15:11:25.303777+0000","last_became_active":"2024-09-13T15:10:04.878965+0000","last_became_peered":"2024-09-13T15:10:04.878965+0000","last_unstale":"2024-09-13T15:11:25.303777+0000","last_undegraded":"2024-09-13T15:11:25.303777+0000","last_fullsized":"2024-09-13T15:11:25.303777+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:26:58.745084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":183,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177138+0000","last_change":"2024-09-13T14:59:12.707169+0000","last_active":"2024-09-13T15:11:19.177138+0000","last_peered":"2024-09-13T15:11:19.177138+0000","last_clean":"2024-09-13T15:11:19.177138+0000","last_became_active":"2024-09-13T14:59:12.691012+0000","last_became_peered":"2024-09-13T14:59:12.691012+0000","last_unstale":"2024-09-13T15:11:19.177138+0000","last_undegraded":"2024-09-13T15:11:19.177138+0000","last_fullsized":"2024-09-13T15:11:19.177138+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:58:39.056352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"85'149","reported_seq":613,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281116+0000","last_change":"2024-09-13T15:10:56.019979+0000","last_active":"2024-09-13T15:11:19.281116+0000","last_peered":"2024-09-13T15:11:19.281116+0000","last_clean":"2024-09-13T15:11:19.281116+0000","last_became_active":"2024-09-13T15:10:56.018652+0000","last_became_peered":"2024-09-13T15:10:56.018652+0000","last_unstale":"2024-09-13T15:11:19.281116+0000","last_undegraded":"2024-09-13T15:11:19.281116+0000","last_fullsized":"2024-09-13T15:11:19.281116+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:42.265583+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:09:42.265583+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:50:35.758712+0000","scrub_duration":0.0042796969999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":537,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558480+0000","last_change":"2024-09-13T15:10:15.900103+0000","last_active":"2024-09-13T15:11:19.558480+0000","last_peered":"2024-09-13T15:11:19.558480+0000","last_clean":"2024-09-13T15:11:19.558480+0000","last_became_active":"2024-09-13T15:10:15.898923+0000","last_became_peered":"2024-09-13T15:10:15.898923+0000","last_unstale":"2024-09-13T15:11:19.558480+0000","last_undegraded":"2024-09-13T15:11:19.558480+0000","last_fullsized":"2024-09-13T15:11:19.558480+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:08:02.884945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177256+0000","last_change":"2024-09-13T15:09:53.831503+0000","last_active":"2024-09-13T15:11:19.177256+0000","last_peered":"2024-09-13T15:11:19.177256+0000","last_clean":"2024-09-13T15:11:19.177256+0000","last_became_active":"2024-09-13T15:09:53.831109+0000","last_became_peered":"2024-09-13T15:09:53.831109+0000","last_unstale":"2024-09-13T15:11:19.177256+0000","last_undegraded":"2024-09-13T15:11:19.177256+0000","last_fullsized":"2024-09-13T15:11:19.177256+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:16:41.523963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.176864+0000","last_change":"2024-09-13T15:09:53.833657+0000","last_active":"2024-09-13T15:11:19.176864+0000","last_peered":"2024-09-13T15:11:19.176864+0000","last_clean":"2024-09-13T15:11:19.176864+0000","last_became_active":"2024-09-13T15:09:53.833270+0000","last_became_peered":"2024-09-13T15:09:53.833270+0000","last_unstale":"2024-09-13T15:11:19.176864+0000","last_undegraded":"2024-09-13T15:11:19.176864+0000","last_fullsized":"2024-09-13T15:11:19.176864+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:34:34.971883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177492+0000","last_change":"2024-09-13T15:10:15.900211+0000","last_active":"2024-09-13T15:11:19.177492+0000","last_peered":"2024-09-13T15:11:19.177492+0000","last_clean":"2024-09-13T15:11:19.177492+0000","last_became_active":"2024-09-13T15:10:15.899811+0000","last_became_peered":"2024-09-13T15:10:15.899811+0000","last_unstale":"2024-09-13T15:11:19.177492+0000","last_undegraded":"2024-09-13T15:11:19.177492+0000","last_fullsized":"2024-09-13T15:11:19.177492+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:39:18.102530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"85'149","reported_seq":613,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281254+0000","last_change":"2024-09-13T15:10:56.021400+0000","last_active":"2024-09-13T15:11:19.281254+0000","last_peered":"2024-09-13T15:11:19.281254+0000","last_clean":"2024-09-13T15:11:19.281254+0000","last_became_active":"2024-09-13T15:10:56.021067+0000","last_became_peered":"2024-09-13T15:10:56.021067+0000","last_unstale":"2024-09-13T15:11:19.281254+0000","last_undegraded":"2024-09-13T15:11:19.281254+0000","last_fullsized":"2024-09-13T15:11:19.281254+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:57.442552+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T15:09:57.442552+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:49:08.584114+0000","scrub_duration":0.0084266059999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":847,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304136+0000","last_change":"2024-09-13T15:10:15.899365+0000","last_active":"2024-09-13T15:11:25.304136+0000","last_peered":"2024-09-13T15:11:25.304136+0000","last_clean":"2024-09-13T15:11:25.304136+0000","last_became_active":"2024-09-13T15:10:15.898995+0000","last_became_peered":"2024-09-13T15:10:15.898995+0000","last_unstale":"2024-09-13T15:11:25.304136+0000","last_undegraded":"2024-09-13T15:11:25.304136+0000","last_fullsized":"2024-09-13T15:11:25.304136+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:06:36.712427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190142+0000","last_change":"2024-09-13T15:07:52.946957+0000","last_active":"2024-09-13T15:11:24.190142+0000","last_peered":"2024-09-13T15:11:24.190142+0000","last_clean":"2024-09-13T15:11:24.190142+0000","last_became_active":"2024-09-13T15:07:52.946389+0000","last_became_peered":"2024-09-13T15:07:52.946389+0000","last_unstale":"2024-09-13T15:11:24.190142+0000","last_undegraded":"2024-09-13T15:11:24.190142+0000","last_fullsized":"2024-09-13T15:11:24.190142+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:27:44.963362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"85'149","reported_seq":488,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418530+0000","last_change":"2024-09-13T15:11:15.161189+0000","last_active":"2024-09-13T15:11:19.418530+0000","last_peered":"2024-09-13T15:11:19.418530+0000","last_clean":"2024-09-13T15:11:19.418530+0000","last_became_active":"2024-09-13T15:11:15.160829+0000","last_became_peered":"2024-09-13T15:11:15.160829+0000","last_unstale":"2024-09-13T15:11:19.418530+0000","last_undegraded":"2024-09-13T15:11:19.418530+0000","last_fullsized":"2024-09-13T15:11:19.418530+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:46:23.741625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:46:23.741625+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:02:04.813540+0000","scrub_duration":0.0014951610000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"63'1","reported_seq":359,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281418+0000","last_change":"2024-09-13T15:10:56.024902+0000","last_active":"2024-09-13T15:11:19.281418+0000","last_peered":"2024-09-13T15:11:19.281418+0000","last_clean":"2024-09-13T15:11:19.281418+0000","last_became_active":"2024-09-13T15:10:56.023455+0000","last_became_peered":"2024-09-13T15:10:56.023455+0000","last_unstale":"2024-09-13T15:11:19.281418+0000","last_undegraded":"2024-09-13T15:11:19.281418+0000","last_fullsized":"2024-09-13T15:11:19.281418+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:06:47.209241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075291599999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":805,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304206+0000","last_change":"2024-09-13T14:40:37.186293+0000","last_active":"2024-09-13T15:11:25.304206+0000","last_peered":"2024-09-13T15:11:25.304206+0000","last_clean":"2024-09-13T15:11:25.304206+0000","last_became_active":"2024-09-13T14:40:37.185811+0000","last_became_peered":"2024-09-13T14:40:37.185811+0000","last_unstale":"2024-09-13T15:11:25.304206+0000","last_undegraded":"2024-09-13T15:11:25.304206+0000","last_fullsized":"2024-09-13T15:11:25.304206+0000","mapping_epoch":133,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:47:00.142407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":481,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558638+0000","last_change":"2024-09-13T14:59:12.708636+0000","last_active":"2024-09-13T15:11:19.558638+0000","last_peered":"2024-09-13T15:11:19.558638+0000","last_clean":"2024-09-13T15:11:19.558638+0000","last_became_active":"2024-09-13T14:59:12.708235+0000","last_became_peered":"2024-09-13T14:59:12.708235+0000","last_unstale":"2024-09-13T15:11:19.558638+0000","last_undegraded":"2024-09-13T15:11:19.558638+0000","last_fullsized":"2024-09-13T15:11:19.558638+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:57:35.886453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"85'149","reported_seq":897,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190981+0000","last_change":"2024-09-13T15:07:55.932598+0000","last_active":"2024-09-13T15:11:24.190981+0000","last_peered":"2024-09-13T15:11:24.190981+0000","last_clean":"2024-09-13T15:11:24.190981+0000","last_became_active":"2024-09-13T15:07:55.932277+0000","last_became_peered":"2024-09-13T15:07:55.932277+0000","last_unstale":"2024-09-13T15:11:24.190981+0000","last_undegraded":"2024-09-13T15:11:24.190981+0000","last_fullsized":"2024-09-13T15:11:24.190981+0000","mapping_epoch":425,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":426,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.683984+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.683984+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:25:56.613460+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":586,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418510+0000","last_change":"2024-09-13T15:11:15.160593+0000","last_active":"2024-09-13T15:11:19.418510+0000","last_peered":"2024-09-13T15:11:19.418510+0000","last_clean":"2024-09-13T15:11:19.418510+0000","last_became_active":"2024-09-13T15:11:15.160234+0000","last_became_peered":"2024-09-13T15:11:15.160234+0000","last_unstale":"2024-09-13T15:11:19.418510+0000","last_undegraded":"2024-09-13T15:11:19.418510+0000","last_fullsized":"2024-09-13T15:11:19.418510+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:02:34.038625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026735099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190622+0000","last_change":"2024-09-13T15:10:04.877622+0000","last_active":"2024-09-13T15:11:24.190622+0000","last_peered":"2024-09-13T15:11:24.190622+0000","last_clean":"2024-09-13T15:11:24.190622+0000","last_became_active":"2024-09-13T15:10:04.877252+0000","last_became_peered":"2024-09-13T15:10:04.877252+0000","last_unstale":"2024-09-13T15:11:24.190622+0000","last_undegraded":"2024-09-13T15:11:24.190622+0000","last_fullsized":"2024-09-13T15:11:24.190622+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:45:45.678830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":65,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190527+0000","last_change":"2024-09-13T15:10:26.640729+0000","last_active":"2024-09-13T15:11:24.190527+0000","last_peered":"2024-09-13T15:11:24.190527+0000","last_clean":"2024-09-13T15:11:24.190527+0000","last_became_active":"2024-09-13T15:10:26.640399+0000","last_became_peered":"2024-09-13T15:10:26.640399+0000","last_unstale":"2024-09-13T15:11:24.190527+0000","last_undegraded":"2024-09-13T15:11:24.190527+0000","last_fullsized":"2024-09-13T15:11:24.190527+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:51:22.838751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"85'149","reported_seq":504,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418627+0000","last_change":"2024-09-13T15:11:15.162539+0000","last_active":"2024-09-13T15:11:19.418627+0000","last_peered":"2024-09-13T15:11:19.418627+0000","last_clean":"2024-09-13T15:11:19.418627+0000","last_became_active":"2024-09-13T15:11:15.161972+0000","last_became_peered":"2024-09-13T15:11:15.161972+0000","last_unstale":"2024-09-13T15:11:19.418627+0000","last_undegraded":"2024-09-13T15:11:19.418627+0000","last_fullsized":"2024-09-13T15:11:19.418627+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:51:57.041870+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:51:57.041870+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:46:56.417880+0000","scrub_duration":0.0054109529999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":76,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191660+0000","last_change":"2024-09-13T15:10:04.865457+0000","last_active":"2024-09-13T15:11:24.191660+0000","last_peered":"2024-09-13T15:11:24.191660+0000","last_clean":"2024-09-13T15:11:24.191660+0000","last_became_active":"2024-09-13T15:10:04.865070+0000","last_became_peered":"2024-09-13T15:10:04.865070+0000","last_unstale":"2024-09-13T15:11:24.191660+0000","last_undegraded":"2024-09-13T15:11:24.191660+0000","last_fullsized":"2024-09-13T15:11:24.191660+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:52:01.050688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025091300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":168,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190724+0000","last_change":"2024-09-13T15:09:53.824061+0000","last_active":"2024-09-13T15:11:24.190724+0000","last_peered":"2024-09-13T15:11:24.190724+0000","last_clean":"2024-09-13T15:11:24.190724+0000","last_became_active":"2024-09-13T15:09:53.823713+0000","last_became_peered":"2024-09-13T15:09:53.823713+0000","last_unstale":"2024-09-13T15:11:24.190724+0000","last_undegraded":"2024-09-13T15:11:24.190724+0000","last_fullsized":"2024-09-13T15:11:24.190724+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:13:01.876205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190028+0000","last_change":"2024-09-13T15:10:15.911210+0000","last_active":"2024-09-13T15:11:24.190028+0000","last_peered":"2024-09-13T15:11:24.190028+0000","last_clean":"2024-09-13T15:11:24.190028+0000","last_became_active":"2024-09-13T15:10:15.910792+0000","last_became_peered":"2024-09-13T15:10:15.910792+0000","last_unstale":"2024-09-13T15:11:24.190028+0000","last_undegraded":"2024-09-13T15:11:24.190028+0000","last_fullsized":"2024-09-13T15:11:24.190028+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:18:53.919718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"85'149","reported_seq":662,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281189+0000","last_change":"2024-09-13T15:10:56.018316+0000","last_active":"2024-09-13T15:11:19.281189+0000","last_peered":"2024-09-13T15:11:19.281189+0000","last_clean":"2024-09-13T15:11:19.281189+0000","last_became_active":"2024-09-13T15:10:56.017994+0000","last_became_peered":"2024-09-13T15:10:56.017994+0000","last_unstale":"2024-09-13T15:11:19.281189+0000","last_undegraded":"2024-09-13T15:11:19.281189+0000","last_fullsized":"2024-09-13T15:11:19.281189+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":4,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T15:09:58.408698+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:56.295859+0000","last_clean_scrub_stamp":"2024-09-13T15:09:58.408698+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:03:41.281936+0000","scrub_duration":0.0030994859999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"65'2","reported_seq":360,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281498+0000","last_change":"2024-09-13T15:10:56.021121+0000","last_active":"2024-09-13T15:11:19.281498+0000","last_peered":"2024-09-13T15:11:19.281498+0000","last_clean":"2024-09-13T15:11:19.281498+0000","last_became_active":"2024-09-13T15:10:56.020798+0000","last_became_peered":"2024-09-13T15:10:56.020798+0000","last_unstale":"2024-09-13T15:11:19.281498+0000","last_undegraded":"2024-09-13T15:11:19.281498+0000","last_fullsized":"2024-09-13T15:11:19.281498+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:53:13.477499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073035899999999998,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210653+0000","last_change":"2024-09-13T15:10:04.877070+0000","last_active":"2024-09-13T15:11:24.210653+0000","last_peered":"2024-09-13T15:11:24.210653+0000","last_clean":"2024-09-13T15:11:24.210653+0000","last_became_active":"2024-09-13T15:10:04.876749+0000","last_became_peered":"2024-09-13T15:10:04.876749+0000","last_unstale":"2024-09-13T15:11:24.210653+0000","last_undegraded":"2024-09-13T15:11:24.210653+0000","last_fullsized":"2024-09-13T15:11:24.210653+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:33:48.861437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190631+0000","last_change":"2024-09-13T15:10:15.894882+0000","last_active":"2024-09-13T15:11:24.190631+0000","last_peered":"2024-09-13T15:11:24.190631+0000","last_clean":"2024-09-13T15:11:24.190631+0000","last_became_active":"2024-09-13T15:10:15.894220+0000","last_became_peered":"2024-09-13T15:10:15.894220+0000","last_unstale":"2024-09-13T15:11:24.190631+0000","last_undegraded":"2024-09-13T15:11:24.190631+0000","last_fullsized":"2024-09-13T15:11:24.190631+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:06:59.115938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"85'149","reported_seq":748,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558624+0000","last_change":"2024-09-13T14:42:58.856976+0000","last_active":"2024-09-13T15:11:19.558624+0000","last_peered":"2024-09-13T15:11:19.558624+0000","last_clean":"2024-09-13T15:11:19.558624+0000","last_became_active":"2024-09-13T14:42:56.296540+0000","last_became_peered":"2024-09-13T14:42:56.296540+0000","last_unstale":"2024-09-13T15:11:19.558624+0000","last_undegraded":"2024-09-13T15:11:19.558624+0000","last_fullsized":"2024-09-13T15:11:19.558624+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_deep_scrub":"85'149","last_deep_scrub_stamp":"2024-09-13T14:42:58.856831+0000","last_clean_scrub_stamp":"2024-09-13T14:42:58.856831+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:54:57.828756+0000","scrub_duration":0.0065768140000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":363,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281597+0000","last_change":"2024-09-13T15:10:56.019310+0000","last_active":"2024-09-13T15:11:19.281597+0000","last_peered":"2024-09-13T15:11:19.281597+0000","last_clean":"2024-09-13T15:11:19.281597+0000","last_became_active":"2024-09-13T15:10:56.019013+0000","last_became_peered":"2024-09-13T15:10:56.019013+0000","last_unstale":"2024-09-13T15:11:19.281597+0000","last_undegraded":"2024-09-13T15:11:19.281597+0000","last_fullsized":"2024-09-13T15:11:19.281597+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:22:29.063329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":175,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190846+0000","last_change":"2024-09-13T15:10:04.869226+0000","last_active":"2024-09-13T15:11:24.190846+0000","last_peered":"2024-09-13T15:11:24.190846+0000","last_clean":"2024-09-13T15:11:24.190846+0000","last_became_active":"2024-09-13T15:10:04.868804+0000","last_became_peered":"2024-09-13T15:10:04.868804+0000","last_unstale":"2024-09-13T15:11:24.190846+0000","last_undegraded":"2024-09-13T15:11:24.190846+0000","last_fullsized":"2024-09-13T15:11:24.190846+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:17:46.742018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032281200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190130+0000","last_change":"2024-09-13T15:10:15.911230+0000","last_active":"2024-09-13T15:11:24.190130+0000","last_peered":"2024-09-13T15:11:24.190130+0000","last_clean":"2024-09-13T15:11:24.190130+0000","last_became_active":"2024-09-13T15:10:15.910902+0000","last_became_peered":"2024-09-13T15:10:15.910902+0000","last_unstale":"2024-09-13T15:11:24.190130+0000","last_undegraded":"2024-09-13T15:11:24.190130+0000","last_fullsized":"2024-09-13T15:11:24.190130+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:54:54.995764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"85'149","reported_seq":831,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558631+0000","last_change":"2024-09-13T15:10:07.870238+0000","last_active":"2024-09-13T15:11:19.558631+0000","last_peered":"2024-09-13T15:11:19.558631+0000","last_clean":"2024-09-13T15:11:19.558631+0000","last_became_active":"2024-09-13T15:10:07.869795+0000","last_became_peered":"2024-09-13T15:10:07.869795+0000","last_unstale":"2024-09-13T15:11:19.558631+0000","last_undegraded":"2024-09-13T15:11:19.558631+0000","last_fullsized":"2024-09-13T15:11:19.558631+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:57.907232+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:57.907232+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:02:47.671154+0000","scrub_duration":0.0080479999999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190547+0000","last_change":"2024-09-13T15:10:15.911505+0000","last_active":"2024-09-13T15:11:24.190547+0000","last_peered":"2024-09-13T15:11:24.190547+0000","last_clean":"2024-09-13T15:11:24.190547+0000","last_became_active":"2024-09-13T15:10:15.910951+0000","last_became_peered":"2024-09-13T15:10:15.910951+0000","last_unstale":"2024-09-13T15:11:24.190547+0000","last_undegraded":"2024-09-13T15:11:24.190547+0000","last_fullsized":"2024-09-13T15:11:24.190547+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:41.400873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":256,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418939+0000","last_change":"2024-09-13T15:11:15.163170+0000","last_active":"2024-09-13T15:11:19.418939+0000","last_peered":"2024-09-13T15:11:19.418939+0000","last_clean":"2024-09-13T15:11:19.418939+0000","last_became_active":"2024-09-13T15:11:15.162898+0000","last_became_peered":"2024-09-13T15:11:15.162898+0000","last_unstale":"2024-09-13T15:11:19.418939+0000","last_undegraded":"2024-09-13T15:11:19.418939+0000","last_fullsized":"2024-09-13T15:11:19.418939+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:18:56.176185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028015100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":59,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190337+0000","last_change":"2024-09-13T15:10:04.875459+0000","last_active":"2024-09-13T15:11:24.190337+0000","last_peered":"2024-09-13T15:11:24.190337+0000","last_clean":"2024-09-13T15:11:24.190337+0000","last_became_active":"2024-09-13T15:10:04.874915+0000","last_became_peered":"2024-09-13T15:10:04.874915+0000","last_unstale":"2024-09-13T15:11:24.190337+0000","last_undegraded":"2024-09-13T15:11:24.190337+0000","last_fullsized":"2024-09-13T15:11:24.190337+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T23:34:04.852075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"85'149","reported_seq":711,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210779+0000","last_change":"2024-09-13T15:09:56.647357+0000","last_active":"2024-09-13T15:11:24.210779+0000","last_peered":"2024-09-13T15:11:24.210779+0000","last_clean":"2024-09-13T15:11:24.210779+0000","last_became_active":"2024-09-13T15:09:56.646785+0000","last_became_peered":"2024-09-13T15:09:56.646785+0000","last_unstale":"2024-09-13T15:11:24.210779+0000","last_undegraded":"2024-09-13T15:11:24.210779+0000","last_fullsized":"2024-09-13T15:11:24.210779+0000","mapping_epoch":467,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":468,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:53:14.267474+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:53:14.267474+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:13.336788+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"71'2","reported_seq":860,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303779+0000","last_change":"2024-09-13T15:10:04.888105+0000","last_active":"2024-09-13T15:11:25.303779+0000","last_peered":"2024-09-13T15:11:25.303779+0000","last_clean":"2024-09-13T15:11:25.303779+0000","last_became_active":"2024-09-13T15:10:04.878585+0000","last_became_peered":"2024-09-13T15:10:04.878585+0000","last_unstale":"2024-09-13T15:11:25.303779+0000","last_undegraded":"2024-09-13T15:11:25.303779+0000","last_fullsized":"2024-09-13T15:11:25.303779+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:41:16.156696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191049+0000","last_change":"2024-09-13T15:10:15.904203+0000","last_active":"2024-09-13T15:11:24.191049+0000","last_peered":"2024-09-13T15:11:24.191049+0000","last_clean":"2024-09-13T15:11:24.191049+0000","last_became_active":"2024-09-13T15:10:15.903806+0000","last_became_peered":"2024-09-13T15:10:15.903806+0000","last_unstale":"2024-09-13T15:11:24.191049+0000","last_undegraded":"2024-09-13T15:11:24.191049+0000","last_fullsized":"2024-09-13T15:11:24.191049+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:03:35.485847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":870,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303674+0000","last_change":"2024-09-13T15:10:04.878380+0000","last_active":"2024-09-13T15:11:25.303674+0000","last_peered":"2024-09-13T15:11:25.303674+0000","last_clean":"2024-09-13T15:11:25.303674+0000","last_became_active":"2024-09-13T15:10:04.877985+0000","last_became_peered":"2024-09-13T15:10:04.877985+0000","last_unstale":"2024-09-13T15:11:25.303674+0000","last_undegraded":"2024-09-13T15:11:25.303674+0000","last_fullsized":"2024-09-13T15:11:25.303674+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:34:43.195981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"85'149","reported_seq":1078,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304341+0000","last_change":"2024-09-13T14:42:56.333285+0000","last_active":"2024-09-13T15:11:25.304341+0000","last_peered":"2024-09-13T15:11:25.304341+0000","last_clean":"2024-09-13T15:11:25.304341+0000","last_became_active":"2024-09-13T14:42:56.297295+0000","last_became_peered":"2024-09-13T14:42:56.297295+0000","last_unstale":"2024-09-13T15:11:25.304341+0000","last_undegraded":"2024-09-13T15:11:25.304341+0000","last_fullsized":"2024-09-13T15:11:25.304341+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":146,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":3,"last_scrub":"85'149","last_scrub_stamp":"2024-09-13T14:42:56.333212+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:17:00.645661+0000","last_clean_scrub_stamp":"2024-09-13T14:42:56.333212+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:04:35.102142+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":73,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190701+0000","last_change":"2024-09-13T15:10:15.905192+0000","last_active":"2024-09-13T15:11:24.190701+0000","last_peered":"2024-09-13T15:11:24.190701+0000","last_clean":"2024-09-13T15:11:24.190701+0000","last_became_active":"2024-09-13T15:10:15.899075+0000","last_became_peered":"2024-09-13T15:10:15.899075+0000","last_unstale":"2024-09-13T15:11:24.190701+0000","last_undegraded":"2024-09-13T15:11:24.190701+0000","last_fullsized":"2024-09-13T15:11:24.190701+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:00:28.249717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":503,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.047821+0000","last_change":"2024-09-13T15:09:53.827985+0000","last_active":"2024-09-13T15:11:25.047821+0000","last_peered":"2024-09-13T15:11:25.047821+0000","last_clean":"2024-09-13T15:11:25.047821+0000","last_became_active":"2024-09-13T15:09:53.827316+0000","last_became_peered":"2024-09-13T15:09:53.827316+0000","last_unstale":"2024-09-13T15:11:25.047821+0000","last_undegraded":"2024-09-13T15:11:25.047821+0000","last_fullsized":"2024-09-13T15:11:25.047821+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:19:26.901798+0000","last_clean_scrub_stamp":"2024-09-13T14:19:26.901798+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:29:01.122031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026633899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":184,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210217+0000","last_change":"2024-09-13T14:59:12.709728+0000","last_active":"2024-09-13T15:11:24.210217+0000","last_peered":"2024-09-13T15:11:24.210217+0000","last_clean":"2024-09-13T15:11:24.210217+0000","last_became_active":"2024-09-13T14:59:12.707417+0000","last_became_peered":"2024-09-13T14:59:12.707417+0000","last_unstale":"2024-09-13T15:11:24.210217+0000","last_undegraded":"2024-09-13T15:11:24.210217+0000","last_fullsized":"2024-09-13T15:11:24.210217+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T02:07:16.246280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":219,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177312+0000","last_change":"2024-09-13T15:10:15.900038+0000","last_active":"2024-09-13T15:11:19.177312+0000","last_peered":"2024-09-13T15:11:19.177312+0000","last_clean":"2024-09-13T15:11:19.177312+0000","last_became_active":"2024-09-13T15:10:15.899620+0000","last_became_peered":"2024-09-13T15:10:15.899620+0000","last_unstale":"2024-09-13T15:11:19.177312+0000","last_undegraded":"2024-09-13T15:11:19.177312+0000","last_fullsized":"2024-09-13T15:11:19.177312+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T00:23:20.496234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"85'23","reported_seq":193,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.177139+0000","last_change":"2024-09-13T14:59:12.720870+0000","last_active":"2024-09-13T15:11:19.177139+0000","last_peered":"2024-09-13T15:11:19.177139+0000","last_clean":"2024-09-13T15:11:19.177139+0000","last_became_active":"2024-09-13T14:59:12.707880+0000","last_became_peered":"2024-09-13T14:59:12.707880+0000","last_unstale":"2024-09-13T15:11:19.177139+0000","last_undegraded":"2024-09-13T15:11:19.177139+0000","last_fullsized":"2024-09-13T15:11:19.177139+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:25:05.453591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190267+0000","last_change":"2024-09-13T15:10:15.900173+0000","last_active":"2024-09-13T15:11:24.190267+0000","last_peered":"2024-09-13T15:11:24.190267+0000","last_clean":"2024-09-13T15:11:24.190267+0000","last_became_active":"2024-09-13T15:10:15.899834+0000","last_became_peered":"2024-09-13T15:10:15.899834+0000","last_unstale":"2024-09-13T15:11:24.190267+0000","last_undegraded":"2024-09-13T15:11:24.190267+0000","last_fullsized":"2024-09-13T15:11:24.190267+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:56:06.566755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"71'3","reported_seq":29,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.418828+0000","last_change":"2024-09-13T15:11:15.162666+0000","last_active":"2024-09-13T15:11:19.418828+0000","last_peered":"2024-09-13T15:11:19.418828+0000","last_clean":"2024-09-13T15:11:19.418828+0000","last_became_active":"2024-09-13T15:11:15.162307+0000","last_became_peered":"2024-09-13T15:11:15.162307+0000","last_unstale":"2024-09-13T15:11:19.418828+0000","last_undegraded":"2024-09-13T15:11:19.418828+0000","last_fullsized":"2024-09-13T15:11:19.418828+0000","mapping_epoch":490,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":491,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:51:40.717528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":342,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281538+0000","last_change":"2024-09-13T15:10:56.022799+0000","last_active":"2024-09-13T15:11:19.281538+0000","last_peered":"2024-09-13T15:11:19.281538+0000","last_clean":"2024-09-13T15:11:19.281538+0000","last_became_active":"2024-09-13T15:10:56.022484+0000","last_became_peered":"2024-09-13T15:10:56.022484+0000","last_unstale":"2024-09-13T15:11:19.281538+0000","last_undegraded":"2024-09-13T15:11:19.281538+0000","last_fullsized":"2024-09-13T15:11:19.281538+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:24:12.202093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281200+0000","last_change":"2024-09-13T15:10:56.023490+0000","last_active":"2024-09-13T15:11:19.281200+0000","last_peered":"2024-09-13T15:11:19.281200+0000","last_clean":"2024-09-13T15:11:19.281200+0000","last_became_active":"2024-09-13T15:10:56.022883+0000","last_became_peered":"2024-09-13T15:10:56.022883+0000","last_unstale":"2024-09-13T15:11:19.281200+0000","last_undegraded":"2024-09-13T15:11:19.281200+0000","last_fullsized":"2024-09-13T15:11:19.281200+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:47:22.402212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190201+0000","last_change":"2024-09-13T15:10:04.876037+0000","last_active":"2024-09-13T15:11:24.190201+0000","last_peered":"2024-09-13T15:11:24.190201+0000","last_clean":"2024-09-13T15:11:24.190201+0000","last_became_active":"2024-09-13T15:10:04.875624+0000","last_became_peered":"2024-09-13T15:10:04.875624+0000","last_unstale":"2024-09-13T15:11:24.190201+0000","last_undegraded":"2024-09-13T15:11:24.190201+0000","last_fullsized":"2024-09-13T15:11:24.190201+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:55:12.697390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":844,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303956+0000","last_change":"2024-09-13T15:10:15.898047+0000","last_active":"2024-09-13T15:11:25.303956+0000","last_peered":"2024-09-13T15:11:25.303956+0000","last_clean":"2024-09-13T15:11:25.303956+0000","last_became_active":"2024-09-13T15:10:15.897466+0000","last_became_peered":"2024-09-13T15:10:15.897466+0000","last_unstale":"2024-09-13T15:11:25.303956+0000","last_undegraded":"2024-09-13T15:11:25.303956+0000","last_fullsized":"2024-09-13T15:11:25.303956+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:03.737634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":490,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.558694+0000","last_change":"2024-09-13T15:07:52.936799+0000","last_active":"2024-09-13T15:11:19.558694+0000","last_peered":"2024-09-13T15:11:19.558694+0000","last_clean":"2024-09-13T15:11:19.558694+0000","last_became_active":"2024-09-13T15:07:52.936436+0000","last_became_peered":"2024-09-13T15:07:52.936436+0000","last_unstale":"2024-09-13T15:11:19.558694+0000","last_undegraded":"2024-09-13T15:11:19.558694+0000","last_fullsized":"2024-09-13T15:11:19.558694+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T14:52:09.480999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":913,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304108+0000","last_change":"2024-09-13T15:10:15.894390+0000","last_active":"2024-09-13T15:11:25.304108+0000","last_peered":"2024-09-13T15:11:25.304108+0000","last_clean":"2024-09-13T15:11:25.304108+0000","last_became_active":"2024-09-13T15:10:15.894013+0000","last_became_peered":"2024-09-13T15:10:15.894013+0000","last_unstale":"2024-09-13T15:11:25.304108+0000","last_undegraded":"2024-09-13T15:11:25.304108+0000","last_fullsized":"2024-09-13T15:11:25.304108+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:14:15.836830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":59,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190833+0000","last_change":"2024-09-13T15:10:04.881987+0000","last_active":"2024-09-13T15:11:24.190833+0000","last_peered":"2024-09-13T15:11:24.190833+0000","last_clean":"2024-09-13T15:11:24.190833+0000","last_became_active":"2024-09-13T15:10:04.881601+0000","last_became_peered":"2024-09-13T15:10:04.881601+0000","last_unstale":"2024-09-13T15:11:24.190833+0000","last_undegraded":"2024-09-13T15:11:24.190833+0000","last_fullsized":"2024-09-13T15:11:24.190833+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T16:27:39.286013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":153,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.189994+0000","last_change":"2024-09-13T15:07:52.936887+0000","last_active":"2024-09-13T15:11:24.189994+0000","last_peered":"2024-09-13T15:11:24.189994+0000","last_clean":"2024-09-13T15:11:24.189994+0000","last_became_active":"2024-09-13T15:07:52.936398+0000","last_became_peered":"2024-09-13T15:07:52.936398+0000","last_unstale":"2024-09-13T15:11:24.189994+0000","last_undegraded":"2024-09-13T15:11:24.189994+0000","last_fullsized":"2024-09-13T15:11:24.189994+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T18:09:05.892375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":843,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304298+0000","last_change":"2024-09-13T15:07:52.935989+0000","last_active":"2024-09-13T15:11:25.304298+0000","last_peered":"2024-09-13T15:11:25.304298+0000","last_clean":"2024-09-13T15:11:25.304298+0000","last_became_active":"2024-09-13T15:07:52.935661+0000","last_became_peered":"2024-09-13T15:07:52.935661+0000","last_unstale":"2024-09-13T15:11:25.304298+0000","last_undegraded":"2024-09-13T15:11:25.304298+0000","last_fullsized":"2024-09-13T15:11:25.304298+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T17:57:49.027141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"71'1","reported_seq":219,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210373+0000","last_change":"2024-09-13T15:09:53.829469+0000","last_active":"2024-09-13T15:11:24.210373+0000","last_peered":"2024-09-13T15:11:24.210373+0000","last_clean":"2024-09-13T15:11:24.210373+0000","last_became_active":"2024-09-13T15:09:53.829050+0000","last_became_peered":"2024-09-13T15:09:53.829050+0000","last_unstale":"2024-09-13T15:11:24.210373+0000","last_undegraded":"2024-09-13T15:11:24.210373+0000","last_fullsized":"2024-09-13T15:11:24.210373+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:37:54.177468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":68,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.191781+0000","last_change":"2024-09-13T15:09:53.829973+0000","last_active":"2024-09-13T15:11:24.191781+0000","last_peered":"2024-09-13T15:11:24.191781+0000","last_clean":"2024-09-13T15:11:24.191781+0000","last_became_active":"2024-09-13T15:09:53.829601+0000","last_became_peered":"2024-09-13T15:09:53.829601+0000","last_unstale":"2024-09-13T15:11:24.191781+0000","last_undegraded":"2024-09-13T15:11:24.191781+0000","last_fullsized":"2024-09-13T15:11:24.191781+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:14:47.834948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"71'1","reported_seq":58,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190201+0000","last_change":"2024-09-13T15:10:04.882777+0000","last_active":"2024-09-13T15:11:24.190201+0000","last_peered":"2024-09-13T15:11:24.190201+0000","last_clean":"2024-09-13T15:11:24.190201+0000","last_became_active":"2024-09-13T15:10:04.882416+0000","last_became_peered":"2024-09-13T15:10:04.882416+0000","last_unstale":"2024-09-13T15:11:24.190201+0000","last_undegraded":"2024-09-13T15:11:24.190201+0000","last_fullsized":"2024-09-13T15:11:24.190201+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:36:34.906936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":228,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210537+0000","last_change":"2024-09-13T15:10:15.899883+0000","last_active":"2024-09-13T15:11:24.210537+0000","last_peered":"2024-09-13T15:11:24.210537+0000","last_clean":"2024-09-13T15:11:24.210537+0000","last_became_active":"2024-09-13T15:10:15.899431+0000","last_became_peered":"2024-09-13T15:10:15.899431+0000","last_unstale":"2024-09-13T15:11:24.210537+0000","last_undegraded":"2024-09-13T15:11:24.210537+0000","last_fullsized":"2024-09-13T15:11:24.210537+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T20:02:37.343699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"71'2","reported_seq":370,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281632+0000","last_change":"2024-09-13T15:10:56.021939+0000","last_active":"2024-09-13T15:11:19.281632+0000","last_peered":"2024-09-13T15:11:19.281632+0000","last_clean":"2024-09-13T15:11:19.281632+0000","last_became_active":"2024-09-13T15:10:56.021547+0000","last_became_peered":"2024-09-13T15:10:56.021547+0000","last_unstale":"2024-09-13T15:11:19.281632+0000","last_undegraded":"2024-09-13T15:11:19.281632+0000","last_fullsized":"2024-09-13T15:11:19.281632+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:49:50.175522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":230,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.210303+0000","last_change":"2024-09-13T15:10:04.867225+0000","last_active":"2024-09-13T15:11:24.210303+0000","last_peered":"2024-09-13T15:11:24.210303+0000","last_clean":"2024-09-13T15:11:24.210303+0000","last_became_active":"2024-09-13T15:10:04.866809+0000","last_became_peered":"2024-09-13T15:10:04.866809+0000","last_unstale":"2024-09-13T15:11:24.210303+0000","last_undegraded":"2024-09-13T15:11:24.210303+0000","last_fullsized":"2024-09-13T15:11:24.210303+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T01:17:32.561235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"71'2","reported_seq":355,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:19.281623+0000","last_change":"2024-09-13T15:10:56.023584+0000","last_active":"2024-09-13T15:11:19.281623+0000","last_peered":"2024-09-13T15:11:19.281623+0000","last_clean":"2024-09-13T15:11:19.281623+0000","last_became_active":"2024-09-13T15:10:56.022873+0000","last_became_peered":"2024-09-13T15:10:56.022873+0000","last_unstale":"2024-09-13T15:11:19.281623+0000","last_undegraded":"2024-09-13T15:11:19.281623+0000","last_fullsized":"2024-09-13T15:11:19.281623+0000","mapping_epoch":485,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:56:16.232555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":901,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.303906+0000","last_change":"2024-09-13T15:10:04.877029+0000","last_active":"2024-09-13T15:11:25.303906+0000","last_peered":"2024-09-13T15:11:25.303906+0000","last_clean":"2024-09-13T15:11:25.303906+0000","last_became_active":"2024-09-13T15:10:04.876676+0000","last_became_peered":"2024-09-13T15:10:04.876676+0000","last_unstale":"2024-09-13T15:11:25.303906+0000","last_undegraded":"2024-09-13T15:11:25.303906+0000","last_fullsized":"2024-09-13T15:11:25.303906+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T19:21:16.624424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"71'1","reported_seq":853,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:25.304511+0000","last_change":"2024-09-13T15:09:53.833193+0000","last_active":"2024-09-13T15:11:25.304511+0000","last_peered":"2024-09-13T15:11:25.304511+0000","last_clean":"2024-09-13T15:11:25.304511+0000","last_became_active":"2024-09-13T15:09:53.832777+0000","last_became_peered":"2024-09-13T15:09:53.832777+0000","last_unstale":"2024-09-13T15:11:25.304511+0000","last_undegraded":"2024-09-13T15:11:25.304511+0000","last_fullsized":"2024-09-13T15:11:25.304511+0000","mapping_epoch":464,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":465,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:13:59.972608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":201,"reported_epoch":492,"state":"active+clean","last_fresh":"2024-09-13T15:11:21.922268+0000","last_change":"2024-09-13T15:07:52.933733+0000","last_active":"2024-09-13T15:11:21.922268+0000","last_peered":"2024-09-13T15:11:21.922268+0000","last_clean":"2024-09-13T15:11:21.922268+0000","last_became_active":"2024-09-13T15:07:52.933425+0000","last_became_peered":"2024-09-13T15:07:52.933425+0000","last_unstale":"2024-09-13T15:11:21.922268+0000","last_undegraded":"2024-09-13T15:11:21.922268+0000","last_fullsized":"2024-09-13T15:11:21.922268+0000","mapping_epoch":422,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":423,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:45.753839+0000","last_clean_scrub_stamp":"2024-09-13T14:20:45.753839+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T22:56:59.564486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"71'4","reported_seq":65,"reported_epoch":493,"state":"active+clean","last_fresh":"2024-09-13T15:11:24.190475+0000","last_change":"2024-09-13T15:10:04.887555+0000","last_active":"2024-09-13T15:11:24.190475+0000","last_peered":"2024-09-13T15:11:24.190475+0000","last_clean":"2024-09-13T15:11:24.190475+0000","last_became_active":"2024-09-13T15:10:04.873030+0000","last_became_peered":"2024-09-13T15:10:04.873030+0000","last_unstale":"2024-09-13T15:11:24.190475+0000","last_undegraded":"2024-09-13T15:11:24.190475+0000","last_fullsized":"2024-09-13T15:11:24.190475+0000","mapping_epoch":469,"log_start":"0'0","ondisk_log_start":"0'0","created":67,"last_epoch_clean":470,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T14:20:44.751898+0000","last_clean_scrub_stamp":"2024-09-13T14:20:44.751898+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:27:14.195107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":5954,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18409,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234626,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429772,"kb_used_data":6656,"kb_used_omap":1,"kb_used_meta":1423102,"kb_avail":92315380,"statfs":{"total":95995035648,"available":94530949120,"internally_reserved":0,"allocated":6815744,"data_stored":4232922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":1457256907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":46,"seq":197568496264,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455620,"kb_used_data":6328,"kb_used_omap":1,"kb_used_meta":1449278,"kb_avail":92289532,"statfs":{"total":95995035648,"available":94504480768,"internally_reserved":0,"allocated":6479872,"data_stored":3905115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1484061127},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":38,"seq":163208757902,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700796,"kb_used_data":6896,"kb_used_omap":1,"kb_used_meta":1693886,"kb_avail":92044356,"statfs":{"total":95995035648,"available":94253420544,"internally_reserved":0,"allocated":7061504,"data_stored":4490978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1734539718},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":30,"seq":128849019538,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351560,"kb_used_data":6332,"kb_used_omap":1,"kb_used_meta":1345214,"kb_avail":92393592,"statfs":{"total":95995035648,"available":94611038208,"internally_reserved":0,"allocated":6483968,"data_stored":3905172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1377499590},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":133,"seq":571230650750,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1156536,"kb_used_data":6892,"kb_used_omap":4,"kb_used_meta":1149627,"kb_avail":92588616,"statfs":{"total":95995035648,"available":94810742784,"internally_reserved":0,"allocated":7057408,"data_stored":4490740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4561,"internal_metadata":1177218607},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":126,"seq":541165879685,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1145276,"kb_used_data":6576,"kb_used_omap":6,"kb_used_meta":1138681,"kb_avail":92599876,"statfs":{"total":95995035648,"available":94822273024,"internally_reserved":0,"allocated":6733824,"data_stored":4163586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6599,"internal_metadata":1166009913},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":109,"seq":468151435702,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1321964,"kb_used_data":6304,"kb_used_omap":5,"kb_used_meta":1315642,"kb_avail":92423188,"statfs":{"total":95995035648,"available":94641344512,"internally_reserved":0,"allocated":6455296,"data_stored":3899755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5791,"internal_metadata":1347217761},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":348,"seq":1494648619145,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":114104,"kb_used_data":6316,"kb_used_omap":10,"kb_used_meta":107765,"kb_avail":93631048,"statfs":{"total":95995035648,"available":95878193152,"internally_reserved":0,"allocated":6467584,"data_stored":3900886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10675,"internal_metadata":110351949},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9081,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4197,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2161,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2970,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T15:12:17.433 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-13T15:12:17.434 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi042 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:12:17.629 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=bc62f205628c4381b2e2445577f0361d Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_d58f73069a8157a9ace12a0247ee3e8497707ea9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi042 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-13T15:12:17.632 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T15:12:17.641 INFO:tasks.cephadm:Teardown begin 2024-09-13T15:12:17.642 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:12:17.690 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:12:17.723 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T15:12:17.723 DEBUG:teuthology.orchestra.run.smithi042:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:12:17.748 DEBUG:teuthology.orchestra.run.smithi159:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T15:12:17.796 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T15:12:17.797 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T15:12:17.797 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a 2024-09-13T15:12:18.098 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:17 smithi042 systemd[1]: Stopping Ceph mon.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:18.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:12:18.127+0000 7fcad823d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:18.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a[108539]: 2024-09-13T15:12:18.127+0000 7fcad823d640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T15:12:18.504 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:18 smithi042 podman[363342]: 2024-09-13 15:12:18.190132545 +0000 UTC m=+0.165004796 container died 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True) 2024-09-13T15:12:18.505 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE Bus STOPPING 2024-09-13T15:12:18.505 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:12:18.505 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE Bus STOPPED 2024-09-13T15:12:18.505 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE Bus STARTING 2024-09-13T15:12:18.506 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE Serving on http://:::9283 2024-09-13T15:12:18.506 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:18 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:18] ENGINE Bus STARTED 2024-09-13T15:12:18.848 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:18 smithi042 podman[363342]: 2024-09-13 15:12:18.608778677 +0000 UTC m=+0.583650939 container cleanup 69b5986addfba3c30a1cc390c3fc8f37cf774ffa2e66d3322b143be23b481764 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T15:12:18.849 INFO:journalctl@ceph.mon.a.smithi042.stdout:Sep 13 15:12:18 smithi042 bash[363342]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-a 2024-09-13T15:12:19.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T15:12:19.089 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:19.089 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T15:12:19.090 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T15:12:19.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.c 2024-09-13T15:12:19.598 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:19 smithi042 systemd[1]: Stopping Ceph mon.c for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:19.908 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c[113903]: 2024-09-13T15:12:19.640+0000 7f761e06a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:19.909 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c[113903]: 2024-09-13T15:12:19.640+0000 7f761e06a640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:12:19.909 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE Bus STOPPING 2024-09-13T15:12:19.910 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: ::ffff:172.21.15.159 - - [13/Sep/2024:15:12:19] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-09-13T15:12:19.910 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T15:12:19.910 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE Bus STOPPED 2024-09-13T15:12:19.910 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE Bus STARTING 2024-09-13T15:12:19.911 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE Serving on http://:::9283 2024-09-13T15:12:19.911 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:19 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: [13/Sep/2024:15:12:19] ENGINE Bus STARTED 2024-09-13T15:12:20.208 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:19 smithi042 podman[363514]: 2024-09-13 15:12:19.907657976 +0000 UTC m=+0.588996109 container died 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T15:12:20.583 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:20 smithi042 podman[363514]: 2024-09-13 15:12:20.356245586 +0000 UTC m=+1.037583751 container cleanup 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:20.584 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:20 smithi042 bash[363514]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c 2024-09-13T15:12:20.837 INFO:journalctl@ceph.mon.c.smithi042.stdout:Sep 13 15:12:20 smithi042 podman[363544]: 2024-09-13 15:12:20.582794303 +0000 UTC m=+0.666736158 container remove 251d2b2bf575c9dc2787e11b904ac32e71a39b592bdd1eaa4109258721d5f881 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T15:12:20.842 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T15:12:20.879 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:20.879 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T15:12:20.879 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T15:12:20.879 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b 2024-09-13T15:12:21.204 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:20 smithi159 systemd[1]: Stopping Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b[102496]: 2024-09-13T15:12:21.201+0000 7fd7e2c91640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:21.499 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b[102496]: 2024-09-13T15:12:21.201+0000 7fd7e2c91640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T15:12:21.500 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:21 smithi159 podman[257975]: 2024-09-13 15:12:21.34498915 +0000 UTC m=+0.259875640 container died d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T15:12:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:21 smithi159 podman[257975]: 2024-09-13 15:12:21.797765233 +0000 UTC m=+0.712651725 container cleanup d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:12:21.999 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:21 smithi159 bash[257975]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b 2024-09-13T15:12:22.300 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T15:12:22.327 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:22 smithi159 podman[257992]: 2024-09-13 15:12:22.152288465 +0000 UTC m=+0.807620810 container remove d2744427c0abec28bc68de0c5b4de8310b751d1b237a559c122225aeebb9d742 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mon-b, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:12:22.327 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:22 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T15:12:22.328 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:22 smithi159 systemd[1]: Stopped Ceph mon.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:12:22.328 INFO:journalctl@ceph.mon.b.smithi159.stdout:Sep 13 15:12:22 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mon.b.service: Consumed 24.834s CPU time. 2024-09-13T15:12:22.354 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:22.354 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T15:12:22.354 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T15:12:22.354 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y 2024-09-13T15:12:22.656 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:22 smithi042 systemd[1]: Stopping Ceph mgr.y for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:22.988 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:22 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y[89483]: 2024-09-13T15:12:22.653+0000 7f2441484640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:22.989 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:22 smithi042 podman[363657]: 2024-09-13 15:12:22.686006204 +0000 UTC m=+0.134623618 container died 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T15:12:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:23 smithi042 podman[363657]: 2024-09-13 15:12:23.169567054 +0000 UTC m=+0.618184467 container cleanup 06039da613e43c8ae64724794b8c89af65c659d36cc000ff66bdcb90cc06cfd3 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T15:12:23.348 INFO:journalctl@ceph.mgr.y.smithi042.stdout:Sep 13 15:12:23 smithi042 bash[363657]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-y 2024-09-13T15:12:23.613 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T15:12:23.649 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:23.649 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T15:12:23.650 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T15:12:23.650 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x 2024-09-13T15:12:23.999 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 15:12:23 smithi159 systemd[1]: Stopping Ceph mgr.x for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:24.317 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 15:12:24 smithi159 podman[258116]: 2024-09-13 15:12:24.005068398 +0000 UTC m=+0.158907058 container died 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:24.650 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 15:12:24 smithi159 podman[258116]: 2024-09-13 15:12:24.413935898 +0000 UTC m=+0.567774557 container cleanup 66bfd809a8528e0830617c4aeb05b4db60212ffcf2fa038d045f0151356779b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T15:12:24.650 INFO:journalctl@ceph.mgr.x.smithi159.stdout:Sep 13 15:12:24 smithi159 bash[258116]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-mgr-x 2024-09-13T15:12:24.792 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T15:12:24.828 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:24.828 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T15:12:24.828 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T15:12:24.829 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0 2024-09-13T15:12:25.127 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:24 smithi042 systemd[1]: Stopping Ceph osd.0 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:25.395 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:12:25.125+0000 7f1ce12a6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:25.395 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:12:25.125+0000 7f1ce12a6640 -1 osd.0 493 *** Got signal Terminated *** 2024-09-13T15:12:25.395 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:25 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0[283654]: 2024-09-13T15:12:25.125+0000 7f1ce12a6640 -1 osd.0 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:12:26.499 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.037Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.040Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:12:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:12:26.043Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:12:30.952 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:30 smithi042 podman[363788]: 2024-09-13 15:12:30.673824575 +0000 UTC m=+5.645291020 container died fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-09-13T15:12:31.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:31 smithi042 podman[363788]: 2024-09-13 15:12:31.166647239 +0000 UTC m=+6.138113683 container cleanup fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:12:31.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:31 smithi042 bash[363788]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0 2024-09-13T15:12:31.793 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:31 smithi042 podman[363821]: 2024-09-13 15:12:31.477130124 +0000 UTC m=+0.797439735 container remove fbfd4b4eb60f22486dc7e4728f524be35555c9539bf3ba2302f1d2a2c42b0d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T15:12:32.044 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:31 smithi042 podman[363915]: 2024-09-13 15:12:31.695710833 +0000 UTC m=+0.016608802 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:12:32.045 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:31 smithi042 podman[363915]: 2024-09-13 15:12:31.819783046 +0000 UTC m=+0.140681044 container create b984fc70d0684eb80cad4f04c96f49d1e060f5a866bdd7f60e4942fb5730af8e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:12:32.045 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:32 smithi042 podman[363915]: 2024-09-13 15:12:32.038625639 +0000 UTC m=+0.359523595 container init b984fc70d0684eb80cad4f04c96f49d1e060f5a866bdd7f60e4942fb5730af8e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:12:32.045 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:32 smithi042 podman[363915]: 2024-09-13 15:12:32.04347704 +0000 UTC m=+0.364375009 container start b984fc70d0684eb80cad4f04c96f49d1e060f5a866bdd7f60e4942fb5730af8e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:32.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:32 smithi042 podman[363915]: 2024-09-13 15:12:32.155611363 +0000 UTC m=+0.476509320 container attach b984fc70d0684eb80cad4f04c96f49d1e060f5a866bdd7f60e4942fb5730af8e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD) 2024-09-13T15:12:32.349 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 15:12:32 smithi042 podman[363915]: 2024-09-13 15:12:32.244494152 +0000 UTC m=+0.565392119 container died b984fc70d0684eb80cad4f04c96f49d1e060f5a866bdd7f60e4942fb5730af8e (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:32.874 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T15:12:32.916 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:32.917 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T15:12:32.917 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T15:12:32.917 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.1 2024-09-13T15:12:33.257 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:33 smithi042 systemd[1]: Stopping Ceph osd.1 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:33.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T15:12:33.255+0000 7f8375d99640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:33.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T15:12:33.255+0000 7f8375d99640 -1 osd.1 493 *** Got signal Terminated *** 2024-09-13T15:12:33.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:33 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1[131752]: 2024-09-13T15:12:33.255+0000 7f8375d99640 -1 osd.1 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:12:39.256 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:38 smithi042 podman[364044]: 2024-09-13 15:12:38.945831097 +0000 UTC m=+5.792280476 container stop 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:39.256 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:38 smithi042 podman[364044]: 2024-09-13 15:12:38.9713633 +0000 UTC m=+5.817812679 container died 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux ) 2024-09-13T15:12:39.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:39 smithi042 podman[364044]: 2024-09-13 15:12:39.420641874 +0000 UTC m=+6.267091299 container cleanup 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:39.598 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:39 smithi042 bash[364044]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1 2024-09-13T15:12:39.848 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:39 smithi042 podman[364065]: 2024-09-13 15:12:39.597165542 +0000 UTC m=+0.641026565 container remove 3806b8398b080840e104d5c2b760d4c1516bf00a0bc1a0a9c9b6d56e50c96131 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T15:12:40.285 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:39.935147489 +0000 UTC m=+0.016570999 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:12:40.285 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:40.039820923 +0000 UTC m=+0.121244432 container create da217ad214bb4edaf9ea2f0048e8fec94746c3fec09b9d5e0a1fd55b28e5ceae (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD) 2024-09-13T15:12:40.286 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:40.284281852 +0000 UTC m=+0.365705362 container init da217ad214bb4edaf9ea2f0048e8fec94746c3fec09b9d5e0a1fd55b28e5ceae (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:12:40.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:40.289723066 +0000 UTC m=+0.371146581 container start da217ad214bb4edaf9ea2f0048e8fec94746c3fec09b9d5e0a1fd55b28e5ceae (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T15:12:40.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:40.409259561 +0000 UTC m=+0.490683071 container attach da217ad214bb4edaf9ea2f0048e8fec94746c3fec09b9d5e0a1fd55b28e5ceae (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:12:40.599 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 15:12:40 smithi042 podman[364162]: 2024-09-13 15:12:40.480032065 +0000 UTC m=+0.561455577 container died da217ad214bb4edaf9ea2f0048e8fec94746c3fec09b9d5e0a1fd55b28e5ceae (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T15:12:41.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T15:12:41.251 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:41.251 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T15:12:41.251 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T15:12:41.252 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.2 2024-09-13T15:12:41.598 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:41 smithi042 systemd[1]: Stopping Ceph osd.2 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:42.098 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T15:12:41.601+0000 7f844b988640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:42.099 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T15:12:41.601+0000 7f844b988640 -1 osd.2 493 *** Got signal Terminated *** 2024-09-13T15:12:42.099 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:41 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2[139731]: 2024-09-13T15:12:41.601+0000 7f844b988640 -1 osd.2 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:12:47.443 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:47 smithi042 podman[364288]: 2024-09-13 15:12:47.153097167 +0000 UTC m=+5.678566991 container died 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:47.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:47 smithi042 podman[364288]: 2024-09-13 15:12:47.632604896 +0000 UTC m=+6.158074765 container cleanup 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T15:12:47.848 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:47 smithi042 bash[364288]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2 2024-09-13T15:12:48.217 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:47 smithi042 podman[364311]: 2024-09-13 15:12:47.900961654 +0000 UTC m=+0.745487141 container remove 247a1eeed07e6d641957615b711f2795a114401f410e55b53156d1041a93601a (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-aclamk-cbt-combined-d58f730, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:12:48.497 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.119759644 +0000 UTC m=+0.016645383 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:12:48.497 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.227035952 +0000 UTC m=+0.123921730 container create bffb09f34bd1d2ed45a7606432e012d8d7095e8041d92b6e36fc6789cb0ef249 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-13T15:12:48.497 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.412680332 +0000 UTC m=+0.309566068 container init bffb09f34bd1d2ed45a7606432e012d8d7095e8041d92b6e36fc6789cb0ef249 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-aclamk-cbt-combined-d58f730, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:12:48.497 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.417614783 +0000 UTC m=+0.314500522 container start bffb09f34bd1d2ed45a7606432e012d8d7095e8041d92b6e36fc6789cb0ef249 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T15:12:48.497 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.495751829 +0000 UTC m=+0.392637573 container attach bffb09f34bd1d2ed45a7606432e012d8d7095e8041d92b6e36fc6789cb0ef249 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T15:12:48.849 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 15:12:48 smithi042 podman[364406]: 2024-09-13 15:12:48.611624932 +0000 UTC m=+0.508510674 container died bffb09f34bd1d2ed45a7606432e012d8d7095e8041d92b6e36fc6789cb0ef249 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T15:12:49.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T15:12:49.324 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:49.325 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T15:12:49.325 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T15:12:49.325 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.3 2024-09-13T15:12:49.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:49 smithi042 systemd[1]: Stopping Ceph osd.3 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:50.098 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T15:12:49.679+0000 7f5dba793640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:50.099 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T15:12:49.679+0000 7f5dba793640 -1 osd.3 493 *** Got signal Terminated *** 2024-09-13T15:12:50.099 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:49 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T15:12:49.679+0000 7f5dba793640 -1 osd.3 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:12:53.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:53 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T15:12:53.132+0000 7f5db6da8640 -1 osd.3 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.711706+0000 front 2024-09-13T15:12:29.711488+0000 (oldest deadline 2024-09-13T15:12:52.611624+0000) 2024-09-13T15:12:53.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:53 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:53.385+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:54.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:54 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3[144531]: 2024-09-13T15:12:54.150+0000 7f5db6da8640 -1 osd.3 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.711706+0000 front 2024-09-13T15:12:29.711488+0000 (oldest deadline 2024-09-13T15:12:52.611624+0000) 2024-09-13T15:12:54.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:54 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:54.391+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:55.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:55.028+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:12:55.598 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:55 smithi042 podman[364535]: 2024-09-13 15:12:55.29706316 +0000 UTC m=+5.744126616 container died 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T15:12:55.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:55 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:55.363+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:56.098 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:55 smithi042 podman[364535]: 2024-09-13 15:12:55.71931786 +0000 UTC m=+6.166381305 container cleanup 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:12:56.099 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:55 smithi042 bash[364535]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3 2024-09-13T15:12:56.099 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:55 smithi042 podman[364557]: 2024-09-13 15:12:55.937233806 +0000 UTC m=+0.636338425 container remove 7530fa88779f937239a7912740bcbe5b1ae0c022fb554447ab2bed1ecdce4c19 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-13T15:12:56.381 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:56.025+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:12:56.381 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:56.008+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:12:56.674 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.253274952 +0000 UTC m=+0.016412250 image pull fec07016a2b0bf363d0b69240081049b65cf13f1c44f6a7196cf8309aeb428cd quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f 2024-09-13T15:12:56.675 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.363178493 +0000 UTC m=+0.126315795 container create d0b6f2c1cc8e0b3227bc23824a763ff9352af2660e26e7a5f991286240702a77 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730) 2024-09-13T15:12:56.675 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.582351782 +0000 UTC m=+0.345489086 container init d0b6f2c1cc8e0b3227bc23824a763ff9352af2660e26e7a5f991286240702a77 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T15:12:56.675 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.587749932 +0000 UTC m=+0.350887238 container start d0b6f2c1cc8e0b3227bc23824a763ff9352af2660e26e7a5f991286240702a77 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-aclamk-cbt-combined-d58f730, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T15:12:56.675 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.673525236 +0000 UTC m=+0.436662723 container attach d0b6f2c1cc8e0b3227bc23824a763ff9352af2660e26e7a5f991286240702a77 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-aclamk-cbt-combined-d58f730, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T15:12:56.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:56.379+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:56.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:12:56.477+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:12:57.078 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 15:12:56 smithi042 podman[364649]: 2024-09-13 15:12:56.782294698 +0000 UTC m=+0.545432007 container died d0b6f2c1cc8e0b3227bc23824a763ff9352af2660e26e7a5f991286240702a77 (image=quay.ceph.io/ceph-ci/ceph@sha256:914e971332a58dba507db28f0971f6f283c67edecc2d9a6537eea399760b8c1f, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-aclamk-cbt-combined-d58f730, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T15:12:57.249 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:56.982+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:12:57.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:56 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:56.991+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:12:57.434 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T15:12:57.477 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:12:57.477 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T15:12:57.477 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T15:12:57.478 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.4 2024-09-13T15:12:57.749 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:57 smithi159 systemd[1]: Stopping Ceph osd.4 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:12:57.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:12:57.477+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:12:57.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:57.345+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:58.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:58 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:58.001+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:12:58.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:57.793+0000 7f9e9e1c2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:12:58.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:57.793+0000 7f9e9e1c2640 -1 osd.4 493 *** Got signal Terminated *** 2024-09-13T15:12:58.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:57.793+0000 7f9e9e1c2640 -1 osd.4 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:12:58.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:57 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:57.941+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:12:58.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:12:58 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:12:58.464+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:12:58.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:58 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:58.343+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:12:59.249 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:58 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:58.896+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:12:59.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:59 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:59.017+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:12:59.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:12:59 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:12:59.443+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:12:59.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:12:59 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:12:59.380+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:00.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:12:59 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:12:59.988+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:00.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:12:59 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:12:59.925+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:13:00.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:00 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:00.397+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:00.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:00 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:00.397+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:00.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:00 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:00.397+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:01.249 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:00 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:13:00.944+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:13:01.249 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:00 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:13:00.944+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.119145+0000 front 2024-09-13T15:12:35.119091+0000 (oldest deadline 2024-09-13T15:13:00.418942+0000) 2024-09-13T15:13:01.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:01.025+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:01.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:01.025+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:01.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:01.406+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:01.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:01.358+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:01.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:01.358+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:02.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:01.997+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:02.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:01.997+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:02.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:13:01.932+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.218387+0000 front 2024-09-13T15:12:29.218577+0000 (oldest deadline 2024-09-13T15:12:55.118475+0000) 2024-09-13T15:13:02.250 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:01 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4[38709]: 2024-09-13T15:13:01.932+0000 7f9e9a7d6640 -1 osd.4 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.119145+0000 front 2024-09-13T15:12:35.119091+0000 (oldest deadline 2024-09-13T15:13:00.418942+0000) 2024-09-13T15:13:02.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:02 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:02.424+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:02.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:02 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:02.399+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:02.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:02 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:02.399+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:03.360 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:03.029+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:03.361 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:03.029+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:03.679 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:03 smithi159 podman[258260]: 2024-09-13 15:13:03.377713274 +0000 UTC m=+5.697638748 container died 9e81d07130677187da0923a87066f3b9e887614db1be83bd916ff33aec61881e (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T15:13:03.679 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:03.359+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:03.679 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:03.359+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:03.680 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:03.396+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:03.680 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:03 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:03.396+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:03.999 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:03 smithi159 podman[258260]: 2024-09-13 15:13:03.774504494 +0000 UTC m=+6.094429958 container cleanup 9e81d07130677187da0923a87066f3b9e887614db1be83bd916ff33aec61881e (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, org.label-schema.license=GPLv2) 2024-09-13T15:13:04.000 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:03 smithi159 bash[258260]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4 2024-09-13T15:13:04.352 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:04.014+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:04.352 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:04.014+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:04.353 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258281]: 2024-09-13 15:13:04.03434207 +0000 UTC m=+0.655452373 container remove 9e81d07130677187da0923a87066f3b9e887614db1be83bd916ff33aec61881e (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=reef-2c01458, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T15:13:04.639 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:04.433+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:04.639 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:04.433+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:04.640 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.254275947 +0000 UTC m=+0.016129198 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T15:13:04.640 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.410549985 +0000 UTC m=+0.172403228 container create 58631845b36d3c1e1eb83eea4748176b4d35eb2748d71fa051710c6fb204f950 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=reef-2c01458, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T15:13:04.640 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:04.398+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:04.640 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:04 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:04.398+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:04.999 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.638329014 +0000 UTC m=+0.400182258 container init 58631845b36d3c1e1eb83eea4748176b4d35eb2748d71fa051710c6fb204f950 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=reef-2c01458, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T15:13:04.999 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.643534004 +0000 UTC m=+0.405387253 container start 58631845b36d3c1e1eb83eea4748176b4d35eb2748d71fa051710c6fb204f950 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T15:13:04.999 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.738053393 +0000 UTC m=+0.499906647 container attach 58631845b36d3c1e1eb83eea4748176b4d35eb2748d71fa051710c6fb204f950 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-2c01458, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:13:05.249 INFO:journalctl@ceph.osd.4.smithi159.stdout:Sep 13 15:13:04 smithi159 podman[258377]: 2024-09-13 15:13:04.998432533 +0000 UTC m=+0.760285785 container died 58631845b36d3c1e1eb83eea4748176b4d35eb2748d71fa051710c6fb204f950 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T15:13:05.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:05.003+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:05.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:05.003+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:05.576 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:05.481+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:05.576 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:05.481+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:05.577 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:05.415+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:05.577 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:05.415+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:05.607 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T15:13:05.653 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:13:05.653 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T15:13:05.654 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T15:13:05.654 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.5 2024-09-13T15:13:05.956 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 systemd[1]: Stopping Ceph osd.5 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:05.957 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:05.955+0000 7fb9cb180640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:13:05.957 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:05.955+0000 7fb9cb180640 -1 osd.5 493 *** Got signal Terminated *** 2024-09-13T15:13:06.249 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:05 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:05.955+0000 7fb9cb180640 -1 osd.5 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:13:06.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:06.012+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:06.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:06.012+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:06.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:06.435+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:06.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:06.435+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:06.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:06.530+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:06.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:06 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:06.530+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:07.451 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:07.046+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:07.452 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:07.046+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:07.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:07.485+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:07.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:07.485+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:07.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:07.450+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:07.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:07.450+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:07.750 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:07 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:07.450+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:44.892626+0000 front 2024-09-13T15:12:44.892599+0000 (oldest deadline 2024-09-13T15:13:07.192469+0000) 2024-09-13T15:13:08.469 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:08.079+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:08.469 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:08.079+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:08.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:08.468+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:08.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:08.468+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:08.749 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:08.468+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:44.892626+0000 front 2024-09-13T15:12:44.892599+0000 (oldest deadline 2024-09-13T15:13:07.192469+0000) 2024-09-13T15:13:08.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:08.534+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:08.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:08.534+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:08.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:08 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:08.534+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:09.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:09.112+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:09.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:09.112+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:09.999 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:09.516+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:10.000 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:09.516+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:10.000 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:09.516+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:44.892626+0000 front 2024-09-13T15:12:44.892599+0000 (oldest deadline 2024-09-13T15:13:07.192469+0000) 2024-09-13T15:13:10.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:09.543+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:10.001 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:09.543+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:10.001 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:09 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:09.543+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:10.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:10.095+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:10.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:10.095+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:10.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:10.095+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:10.999 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:10.528+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:24.990255+0000 front 2024-09-13T15:12:24.990274+0000 (oldest deadline 2024-09-13T15:12:55.590669+0000) 2024-09-13T15:13:11.000 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:10.528+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.591201+0000 front 2024-09-13T15:12:35.591091+0000 (oldest deadline 2024-09-13T15:13:00.291100+0000) 2024-09-13T15:13:11.000 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5[44988]: 2024-09-13T15:13:10.528+0000 7fb9c378c640 -1 osd.5 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:44.892626+0000 front 2024-09-13T15:12:44.892599+0000 (oldest deadline 2024-09-13T15:13:07.192469+0000) 2024-09-13T15:13:11.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:10.585+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:11.001 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:10.585+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:11.001 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:10 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:10.585+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:11.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:11.097+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:11.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:11.097+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:11.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:11.097+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:11.796 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:11 smithi159 podman[258503]: 2024-09-13 15:13:11.507225951 +0000 UTC m=+5.659060984 container died 56133b64527d684fa7646160d1311d86dcd51647de91751cf0dbf33e8c52777a (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:13:11.797 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:11.595+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:11.797 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:11.595+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:11.797 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:11 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:11.595+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:12.055 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:11 smithi159 podman[258503]: 2024-09-13 15:13:11.928390916 +0000 UTC m=+6.080225956 container cleanup 56133b64527d684fa7646160d1311d86dcd51647de91751cf0dbf33e8c52777a (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-13T15:13:12.056 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:11 smithi159 bash[258503]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5 2024-09-13T15:13:12.499 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258523]: 2024-09-13 15:13:12.163038784 +0000 UTC m=+0.653446604 container remove 56133b64527d684fa7646160d1311d86dcd51647de91751cf0dbf33e8c52777a (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-2c01458, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T15:13:12.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:12.053+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:12.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:12.053+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:12.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:12.053+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:12.876 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:12.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:12.876 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:12.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:12.876 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:12 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:12.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:12.877 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258620]: 2024-09-13 15:13:12.478449691 +0000 UTC m=+0.016014213 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T15:13:12.877 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258620]: 2024-09-13 15:13:12.613927454 +0000 UTC m=+0.151491977 container create b2d950baa8809012ea5890bff33e5f168f8db8f3b5b622dcd707a6ea3e6cfec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-13T15:13:12.877 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258620]: 2024-09-13 15:13:12.875169336 +0000 UTC m=+0.412733858 container init b2d950baa8809012ea5890bff33e5f168f8db8f3b5b622dcd707a6ea3e6cfec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, GIT_BRANCH=HEAD) 2024-09-13T15:13:13.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:13.100+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:13.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:13.100+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:13.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:13.100+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:13.250 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258620]: 2024-09-13 15:13:12.8809245 +0000 UTC m=+0.418489031 container start b2d950baa8809012ea5890bff33e5f168f8db8f3b5b622dcd707a6ea3e6cfec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T15:13:13.250 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:12 smithi159 podman[258620]: 2024-09-13 15:13:12.991889329 +0000 UTC m=+0.529453851 container attach b2d950baa8809012ea5890bff33e5f168f8db8f3b5b622dcd707a6ea3e6cfec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458) 2024-09-13T15:13:13.250 INFO:journalctl@ceph.osd.5.smithi159.stdout:Sep 13 15:13:13 smithi159 podman[258620]: 2024-09-13 15:13:13.063091623 +0000 UTC m=+0.600656145 container died b2d950baa8809012ea5890bff33e5f168f8db8f3b5b622dcd707a6ea3e6cfec9 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T15:13:13.656 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T15:13:13.696 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:13:13.697 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T15:13:13.697 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T15:13:13.697 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.6 2024-09-13T15:13:13.999 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:13 smithi159 systemd[1]: Stopping Ceph osd.6 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:13.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:13.649+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:14.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:13.649+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:14.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:13 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:13.649+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:14.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.030+0000 7f09d7240640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:13:14.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.030+0000 7f09d7240640 -1 osd.6 493 *** Got signal Terminated *** 2024-09-13T15:13:14.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.030+0000 7f09d7240640 -1 osd.6 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:13:14.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.055+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:14.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.055+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:14.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:14.055+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:14.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:14.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:15.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:14.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:15.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:14 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:14.642+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:15.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:15.081+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:15.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:15.081+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:15.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:15.081+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:15.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:15.660+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:15.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:15.660+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:15.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:15 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:15.660+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:16.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:16.125+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:16.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:16.125+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:16.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:16.125+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:16.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:16.626+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:16.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:16.626+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:16.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:16 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:16.626+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:17.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:17.163+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:17.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:17.163+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:17.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:17.163+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:17.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:17.620+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:17.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:17.620+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:18.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:17 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:17.620+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:18.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:18.120+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:29.299022+0000 front 2024-09-13T15:12:29.299063+0000 (oldest deadline 2024-09-13T15:12:53.998908+0000) 2024-09-13T15:13:18.499 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:18.120+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:35.700169+0000 front 2024-09-13T15:12:35.699996+0000 (oldest deadline 2024-09-13T15:13:00.999895+0000) 2024-09-13T15:13:18.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:18.120+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.300893+0000 front 2024-09-13T15:12:46.300945+0000 (oldest deadline 2024-09-13T15:13:09.200817+0000) 2024-09-13T15:13:18.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6[50892]: 2024-09-13T15:13:18.120+0000 7f09d3854640 -1 osd.6 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:52.602375+0000 front 2024-09-13T15:12:52.602342+0000 (oldest deadline 2024-09-13T15:13:17.302137+0000) 2024-09-13T15:13:18.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:18.583+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:19.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:18.583+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:19.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:18 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:18.583+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:19.843 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:19 smithi159 podman[258747]: 2024-09-13 15:13:19.591451328 +0000 UTC m=+5.677784192 container died 0aceebef35aef4da041062f46ca441e9641ac54e577f40dd3871711fdaa825dd (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T15:13:19.843 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:19.621+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:19.843 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:19.621+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:19.843 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:19.621+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:19.844 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:19 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:19.621+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:20.249 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:20 smithi159 podman[258747]: 2024-09-13 15:13:20.04016854 +0000 UTC m=+6.126501445 container cleanup 0aceebef35aef4da041062f46ca441e9641ac54e577f40dd3871711fdaa825dd (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731) 2024-09-13T15:13:20.250 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:20 smithi159 bash[258747]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6 2024-09-13T15:13:20.749 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:20 smithi159 podman[258768]: 2024-09-13 15:13:20.442091783 +0000 UTC m=+0.848877367 container remove 0aceebef35aef4da041062f46ca441e9641ac54e577f40dd3871711fdaa825dd (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T15:13:20.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:20.594+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:20.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:20.594+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:20.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:20.594+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:20.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:20 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:20.594+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:21.105 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:20 smithi159 podman[258863]: 2024-09-13 15:13:20.705893346 +0000 UTC m=+0.016606323 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T15:13:21.105 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:20 smithi159 podman[258863]: 2024-09-13 15:13:20.809749157 +0000 UTC m=+0.120462133 container create 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-2c01458, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-13T15:13:21.105 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:21 smithi159 podman[258863]: 2024-09-13 15:13:21.021200736 +0000 UTC m=+0.331913723 container init 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, RELEASE=reef-2c01458, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:13:21.105 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:21 smithi159 podman[258863]: 2024-09-13 15:13:21.026449053 +0000 UTC m=+0.337162030 container start 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-2c01458, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T15:13:21.105 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:21 smithi159 podman[258863]: 2024-09-13 15:13:21.103867038 +0000 UTC m=+0.414580023 container attach 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T15:13:21.500 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:21 smithi159 podman[258863]: 2024-09-13 15:13:21.210420902 +0000 UTC m=+0.521133881 container died 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T15:13:21.750 INFO:journalctl@ceph.osd.6.smithi159.stdout:Sep 13 15:13:21 smithi159 podman[258906]: 2024-09-13 15:13:21.732454537 +0000 UTC m=+0.513137761 container remove 7e037c28c09dcd1d966081a66bb68d8ee60169eee1e06c24ed5a8d6a40e32dce (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:13:21.751 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:21.552+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:21.751 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:21.552+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:21.751 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:21.552+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:21.751 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:21 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:21.552+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:21.756 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T15:13:21.795 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:13:21.795 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T15:13:21.795 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T15:13:21.796 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.7 2024-09-13T15:13:22.121 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:21 smithi159 systemd[1]: Stopping Ceph osd.7 for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:22.122 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.119+0000 7f6fd6234640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T15:13:22.499 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.119+0000 7f6fd6234640 -1 osd.7 493 *** Got signal Terminated *** 2024-09-13T15:13:22.499 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.119+0000 7f6fd6234640 -1 osd.7 493 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T15:13:22.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.538+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:22.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.538+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:22.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.538+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:22.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:22 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:22.538+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:23.900 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:23.501+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:23.901 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:23.501+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:23.901 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:23.501+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:23.901 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:23 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:23.501+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:24.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:24.482+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:24.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:24.482+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:24.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:24.482+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:24.750 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:24 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:24.482+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:25.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:25 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:25.511+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:26.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:25 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:25.511+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:26.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:25 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:25.511+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:26.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:25 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:25.511+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:26.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:25 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:25.511+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.159:6804 osd.4 since back 2024-09-13T15:12:58.633020+0000 front 2024-09-13T15:12:58.632924+0000 (oldest deadline 2024-09-13T15:13:24.532947+0000) 2024-09-13T15:13:26.499 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.037Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.040Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.042Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.500 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:26.043Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.42:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.42:8765: connect: connection refused" 2024-09-13T15:13:26.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:26.547+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T15:12:27.228701+0000 front 2024-09-13T15:12:27.228668+0000 (oldest deadline 2024-09-13T15:12:52.528666+0000) 2024-09-13T15:13:27.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:26.547+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T15:12:37.730246+0000 front 2024-09-13T15:12:37.730234+0000 (oldest deadline 2024-09-13T15:13:02.430089+0000) 2024-09-13T15:13:27.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:26.547+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T15:12:46.431425+0000 front 2024-09-13T15:12:46.431400+0000 (oldest deadline 2024-09-13T15:13:08.131247+0000) 2024-09-13T15:13:27.000 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:26.547+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T15:12:54.532734+0000 front 2024-09-13T15:12:54.532707+0000 (oldest deadline 2024-09-13T15:13:18.632627+0000) 2024-09-13T15:13:27.001 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:26 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7[57226]: 2024-09-13T15:13:26.547+0000 7f6fd2848640 -1 osd.7 493 heartbeat_check: no reply from 172.21.15.159:6804 osd.4 since back 2024-09-13T15:12:58.633020+0000 front 2024-09-13T15:12:58.632924+0000 (oldest deadline 2024-09-13T15:13:24.532947+0000) 2024-09-13T15:13:27.999 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:27 smithi159 podman[258990]: 2024-09-13 15:13:27.709448526 +0000 UTC m=+5.690068992 container died 386bd79b68662ba96fd833f2beafac5652917b0ed5751c0c84f4accc5efc2887 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-13T15:13:28.446 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:28 smithi159 podman[258990]: 2024-09-13 15:13:28.176785588 +0000 UTC m=+6.157406035 container cleanup 386bd79b68662ba96fd833f2beafac5652917b0ed5751c0c84f4accc5efc2887 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:13:28.446 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:28 smithi159 bash[258990]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7 2024-09-13T15:13:28.749 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:28 smithi159 podman[259024]: 2024-09-13 15:13:28.444882729 +0000 UTC m=+0.732555220 container remove 386bd79b68662ba96fd833f2beafac5652917b0ed5751c0c84f4accc5efc2887 (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-2c01458, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T15:13:29.091 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:28 smithi159 podman[259118]: 2024-09-13 15:13:28.699067072 +0000 UTC m=+0.016071853 image pull c129f5a2fd4eccce3e6eec43830eff1929af98bc48f3b850cf144ce5b326e8b5 quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11 2024-09-13T15:13:29.091 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:28 smithi159 podman[259118]: 2024-09-13 15:13:28.804585626 +0000 UTC m=+0.121590444 container create 9dbb1efabdabf4f1315946277b08f46d4364d6f91b8ef8a54d9f3c135a69a59c (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=reef-2c01458, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T15:13:29.091 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:29 smithi159 podman[259118]: 2024-09-13 15:13:29.006694845 +0000 UTC m=+0.323699620 container init 9dbb1efabdabf4f1315946277b08f46d4364d6f91b8ef8a54d9f3c135a69a59c (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T15:13:29.091 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:29 smithi159 podman[259118]: 2024-09-13 15:13:29.011500954 +0000 UTC m=+0.328505727 container start 9dbb1efabdabf4f1315946277b08f46d4364d6f91b8ef8a54d9f3c135a69a59c (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T15:13:29.091 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:29 smithi159 podman[259118]: 2024-09-13 15:13:29.089959264 +0000 UTC m=+0.406964059 container attach 9dbb1efabdabf4f1315946277b08f46d4364d6f91b8ef8a54d9f3c135a69a59c (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T15:13:29.486 INFO:journalctl@ceph.osd.7.smithi159.stdout:Sep 13 15:13:29 smithi159 podman[259118]: 2024-09-13 15:13:29.199744251 +0000 UTC m=+0.516749036 container died 9dbb1efabdabf4f1315946277b08f46d4364d6f91b8ef8a54d9f3c135a69a59c (image=quay.ceph.io/ceph-ci/ceph@sha256:c37c250ffe653639d7a4124ac8acdc3c8f6a53cb4ef1f26b1d2df36648631a11, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-2c01458, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-13T15:13:29.772 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T15:13:29.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:13:29.811 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T15:13:29.811 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T15:13:29.811 DEBUG:teuthology.orchestra.run.smithi159:> sudo systemctl stop ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T15:13:29.999 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:29 smithi159 systemd[1]: Stopping Ceph prometheus.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:30.442 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.131Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T15:13:30.443 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-13T15:13:30.444 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.132Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-13T15:13:30.444 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.135Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T15:13:30.444 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.135Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-13T15:13:30.444 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a[88605]: ts=2024-09-13T15:13:30.135Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-13T15:13:30.445 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 podman[259245]: 2024-09-13 15:13:30.142316914 +0000 UTC m=+0.107095285 container died e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:30.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 podman[259245]: 2024-09-13 15:13:30.621789278 +0000 UTC m=+0.586567635 container cleanup e6c37503183bf4dfee0f7231ab2b96101a492150d611ebbb7eee6f1b15bd7d5e (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:30.749 INFO:journalctl@ceph.prometheus.a.smithi159.stdout:Sep 13 15:13:30 smithi159 bash[259245]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T15:13:31.061 DEBUG:teuthology.orchestra.run.smithi159:> sudo pkill -f 'journalctl -f -n 0 -u ceph-4c2088c8-71da-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T15:13:31.095 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T15:13:31.095 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T15:13:31.095 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:13:31.258 INFO:teuthology.orchestra.run.smithi042.stdout:Deleting cluster with fsid: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T15:13:32.866 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:32 smithi042 systemd[1]: Stopping Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:33.197 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:32 smithi042 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a[103469]: ts=2024-09-13T15:13:32.864Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T15:13:33.198 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:32 smithi042 podman[365076]: 2024-09-13 15:13:32.905976787 +0000 UTC m=+0.150804093 container died 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:33.590 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 podman[365076]: 2024-09-13 15:13:33.395836324 +0000 UTC m=+0.640663631 container cleanup 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:33.591 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 bash[365076]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T15:13:33.848 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 podman[365094]: 2024-09-13 15:13:33.697279587 +0000 UTC m=+0.785447710 container remove 9d75438ff3dbdb7749edd06cb384079add6595062acbfcfba5276e4fb035b095 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:33.849 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 podman[365094]: 2024-09-13 15:13:33.781513693 +0000 UTC m=+0.869681816 volume remove 546391c59637fc7eff33da18f429f6b660a449f7e3bbddbda0611321289a4d50 2024-09-13T15:13:34.348 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:34 smithi042 systemd[1]: Stopping Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:34.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T15:13:34.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 systemd[1]: Stopped Ceph alertmanager.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:13:34.349 INFO:journalctl@ceph.alertmanager.a.smithi042.stdout:Sep 13 15:13:33 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 4.254s CPU time. 2024-09-13T15:13:34.661 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:34 smithi042 podman[365218]: 2024-09-13 15:13:34.414130991 +0000 UTC m=+0.120629087 container died 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:35.045 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:34 smithi042 podman[365218]: 2024-09-13 15:13:34.834436231 +0000 UTC m=+0.540934366 container cleanup 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:35.045 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:34 smithi042 bash[365218]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T15:13:35.348 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:35 smithi042 podman[365236]: 2024-09-13 15:13:35.043965959 +0000 UTC m=+0.629057235 container remove 9c4447f8ffceabc4f56779f8b44e2b04d3112b800ec117a16775bee35b99bd35 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-13T15:13:35.348 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:35 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:13:35.349 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:35 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T15:13:35.349 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:35 smithi042 systemd[1]: Stopped Ceph node-exporter.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:13:35.349 INFO:journalctl@ceph.node-exporter.a.smithi042.stdout:Sep 13 15:13:35 smithi042 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 10.094s CPU time. 2024-09-13T15:13:42.806 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T15:13:42.960 INFO:teuthology.orchestra.run.smithi159.stdout:Deleting cluster with fsid: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T15:13:44.555 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 systemd[1]: Stopping Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:44.884 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=server t=2024-09-13T15:13:44.553574225Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-13T15:13:44.884 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=ticker t=2024-09-13T15:13:44.553965476Z level=info msg=stopped last_tick=2024-09-13T15:13:40Z 2024-09-13T15:13:44.885 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=tracing t=2024-09-13T15:13:44.553969444Z level=info msg="Closing tracing" 2024-09-13T15:13:44.885 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a[93212]: logger=grafana-apiserver t=2024-09-13T15:13:44.553972278Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-13T15:13:44.885 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:44 smithi159 podman[259665]: 2024-09-13 15:13:44.574401421 +0000 UTC m=+0.129468026 container died 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T15:13:45.249 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 podman[259665]: 2024-09-13 15:13:45.05945882 +0000 UTC m=+0.614525456 container cleanup 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T15:13:45.249 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 bash[259665]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a 2024-09-13T15:13:45.646 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 podman[259683]: 2024-09-13 15:13:45.327680905 +0000 UTC m=+0.742567978 container remove 3a8b5443f86c7f54a7b973d7c6313d2d351af01d84e7ce7ed849a271c0900eea (image=quay.io/ceph/grafana:10.4.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-13T15:13:45.646 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-13T15:13:45.647 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 systemd[1]: Stopped Ceph grafana.a for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:13:45.647 INFO:journalctl@ceph.grafana.a.smithi159.stdout:Sep 13 15:13:45 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@grafana.a.service: Consumed 33.077s CPU time. 2024-09-13T15:13:45.902 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:45 smithi159 systemd[1]: Stopping Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968... 2024-09-13T15:13:46.173 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:45 smithi159 podman[259795]: 2024-09-13 15:13:45.901294516 +0000 UTC m=+0.116204750 container died 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:13:46.500 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 podman[259795]: 2024-09-13 15:13:46.363732958 +0000 UTC m=+0.578643231 container cleanup 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:13:46.501 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 bash[259795]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T15:13:46.958 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 podman[259812]: 2024-09-13 15:13:46.639835723 +0000 UTC m=+0.742683678 container remove 4e1a6b244f716ad5d73724e45b268f2ba3d034cbb400291af93aa12f4203a64a (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-4c2088c8-71da-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-13T15:13:46.959 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T15:13:46.959 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T15:13:46.959 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 systemd[1]: Stopped Ceph node-exporter.b for 4c2088c8-71da-11ef-bceb-c7b262605968. 2024-09-13T15:13:46.959 INFO:journalctl@ceph.node-exporter.b.smithi159.stdout:Sep 13 15:13:46 smithi159 systemd[1]: ceph-4c2088c8-71da-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 8.678s CPU time. 2024-09-13T15:13:50.211 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:13:50.245 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T15:13:50.273 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T15:13:50.275 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi042/crash 2024-09-13T15:13:50.276 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:13:50.331 INFO:teuthology.orchestra.run.smithi042.stderr:tar: /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:13:50.331 INFO:teuthology.orchestra.run.smithi042.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:13:50.333 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi159/crash 2024-09-13T15:13:50.334 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash -- . 2024-09-13T15:13:50.403 INFO:teuthology.orchestra.run.smithi159.stderr:tar: /var/lib/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T15:13:50.403 INFO:teuthology.orchestra.run.smithi159.stderr:tar: Error is not recoverable: exiting now 2024-09-13T15:13:50.404 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T15:13:50.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-13T15:13:50.494 INFO:teuthology.orchestra.run.smithi042.stdout:2024-09-13T14:19:29.206405+0000 mon.a (mon.0) 660 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T15:13:50.498 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-13T15:13:50.499 INFO:tasks.cephadm:Compressing logs... 2024-09-13T15:13:50.499 DEBUG:teuthology.orchestra.run.smithi042:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:13:50.502 DEBUG:teuthology.orchestra.run.smithi159:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T15:13:50.529 INFO:teuthology.orchestra.run.smithi042.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T15:13:50.548 INFO:teuthology.orchestra.run.smithi159.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose: No such file or directory -- 2024-09-13T15:13:50.548 INFO:teuthology.orchestra.run.smithi159.stderr: /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T15:13:50.549 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:13:50.549 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T15:13:50.549 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:13:50.549 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:13:50.549 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi159.ypxvst.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log: /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-13T15:13:50.550 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi159.ypxvst.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:13:50.551 INFO:teuthology.orchestra.run.smithi159.stderr: 83.7% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi159.ypxvst.log.gz 2024-09-13T15:13:50.552 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T15:13:50.552 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T15:13:50.552 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T15:13:50.553 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T15:13:50.553 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-13T15:13:50.555 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi042.pkkbnw.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T15:13:50.556 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi042.pkkbnw.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T15:13:50.557 INFO:teuthology.orchestra.run.smithi042.stderr: 86.3% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mds.foo.smithi042.pkkbnw.log.gz 2024-09-13T15:13:50.562 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.8% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:13:50.752 INFO:teuthology.orchestra.run.smithi042.stderr: 94.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:13:50.774 INFO:teuthology.orchestra.run.smithi159.stderr:/var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 85.8% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:13:50.843 INFO:teuthology.orchestra.run.smithi159.stderr: 90.5% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:13:50.867 INFO:teuthology.orchestra.run.smithi159.stderr: 87.2% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T15:13:50.869 INFO:teuthology.orchestra.run.smithi042.stderr: 92.1% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T15:13:50.906 INFO:teuthology.orchestra.run.smithi042.stderr: 91.2% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:13:50.993 INFO:teuthology.orchestra.run.smithi159.stderr: 91.3% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T15:13:51.014 INFO:teuthology.orchestra.run.smithi042.stderr: 94.3% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T15:13:51.394 INFO:teuthology.orchestra.run.smithi159.stderr: 89.8% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-13T15:13:51.547 INFO:teuthology.orchestra.run.smithi159.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T15:13:54.597 INFO:teuthology.orchestra.run.smithi042.stderr: 89.6% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-13T15:14:14.698 INFO:teuthology.orchestra.run.smithi042.stderr: 92.1% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-13T15:14:35.882 INFO:teuthology.orchestra.run.smithi159.stderr: 92.3% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-13T15:14:46.237 INFO:teuthology.orchestra.run.smithi042.stderr: 90.4% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-13T15:29:43.186 INFO:teuthology.orchestra.run.smithi159.stderr: 94.2% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T15:35:53.240 INFO:teuthology.orchestra.run.smithi042.stderr: 93.6% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T15:36:03.639 INFO:teuthology.orchestra.run.smithi159.stderr: 93.5% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T15:40:36.630 INFO:teuthology.orchestra.run.smithi159.stderr: 93.6% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T15:41:22.099 INFO:teuthology.orchestra.run.smithi159.stderr: 92.8% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T15:41:22.101 INFO:teuthology.orchestra.run.smithi159.stderr: 2024-09-13T15:41:22.101 INFO:teuthology.orchestra.run.smithi159.stderr:real 27m31.585s 2024-09-13T15:41:22.101 INFO:teuthology.orchestra.run.smithi159.stderr:user 8m51.589s 2024-09-13T15:41:22.102 INFO:teuthology.orchestra.run.smithi159.stderr:sys 0m29.713s 2024-09-13T15:49:54.389 INFO:teuthology.orchestra.run.smithi042.stderr: 92.6% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T15:53:36.074 INFO:teuthology.orchestra.run.smithi042.stderr: 92.9% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T15:53:37.545 INFO:teuthology.orchestra.run.smithi042.stderr: 92.4% -- replaced with /var/log/ceph/4c2088c8-71da-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T15:53:37.580 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T15:53:37.581 INFO:teuthology.orchestra.run.smithi042.stderr:real 39m47.063s 2024-09-13T15:53:37.581 INFO:teuthology.orchestra.run.smithi042.stderr:user 14m8.866s 2024-09-13T15:53:37.581 INFO:teuthology.orchestra.run.smithi042.stderr:sys 0m39.657s 2024-09-13T15:53:37.581 INFO:tasks.cephadm:Archiving logs... 2024-09-13T15:53:37.583 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi042/log 2024-09-13T15:53:37.584 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:55:45.128 DEBUG:teuthology.misc:Transferring archived files from smithi159:/var/log/ceph to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi159/log 2024-09-13T15:55:45.135 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T15:57:05.461 INFO:tasks.cephadm:Removing cluster... 2024-09-13T15:57:05.462 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 --force 2024-09-13T15:57:06.065 INFO:teuthology.orchestra.run.smithi042.stdout:Deleting cluster with fsid: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T15:57:07.103 DEBUG:teuthology.orchestra.run.smithi159:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 4c2088c8-71da-11ef-bceb-c7b262605968 --force 2024-09-13T15:57:07.713 INFO:teuthology.orchestra.run.smithi159.stdout:Deleting cluster with fsid: 4c2088c8-71da-11ef-bceb-c7b262605968 2024-09-13T15:57:08.553 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T15:57:08.554 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:57:08.574 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T15:57:08.599 INFO:tasks.cephadm:Teardown complete 2024-09-13T15:57:08.599 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T15:57:08.611 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T15:57:08.611 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:57:08.617 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T15:57:08.753 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T15:57:08.762 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T15:57:08.763 DEBUG:teuthology.orchestra.run.smithi042:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:57:08.766 DEBUG:teuthology.orchestra.run.smithi159:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T15:57:08.780 INFO:teuthology.orchestra.run.smithi159.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:57:08.781 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-09-13T15:57:08.825 INFO:teuthology.orchestra.run.smithi159.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:57:08.825 INFO:teuthology.orchestra.run.smithi159.stdout:=============================================================================== 2024-09-13T15:57:08.825 INFO:teuthology.orchestra.run.smithi159.stdout:^* hv01.front.sepia.ceph.com 3 6 377 49 +226us[ +234us] +/- 40ms 2024-09-13T15:57:08.825 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 47 -576us[ -576us] +/- 57ms 2024-09-13T15:57:08.825 INFO:teuthology.orchestra.run.smithi159.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 45 +376us[ +376us] +/- 84ms 2024-09-13T15:57:08.826 INFO:teuthology.orchestra.run.smithi159.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:57:08.861 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T15:57:08.862 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-09-13T15:57:08.862 INFO:teuthology.orchestra.run.smithi042.stdout:^* hv01.front.sepia.ceph.com 3 6 377 45 +229us[ +240us] +/- 40ms 2024-09-13T15:57:08.862 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 44 -611us[ -611us] +/- 57ms 2024-09-13T15:57:08.862 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 110 +406us[ +416us] +/- 83ms 2024-09-13T15:57:08.862 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T15:57:08.870 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T15:57:08.886 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T15:57:08.887 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T15:57:08.896 DEBUG:teuthology.orchestra.run.smithi042:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:57:09.030 DEBUG:teuthology.orchestra.run.smithi159:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T15:57:09.195 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T15:57:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:57:09.227 DEBUG:teuthology.orchestra.run.smithi159:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T15:57:09.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:57:09.257 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T15:57:09.267 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T15:57:09.276 INFO:teuthology.task.internal:Duration was 6619.981971 seconds 2024-09-13T15:57:09.276 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T15:57:09.286 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T15:57:09.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:57:09.289 DEBUG:teuthology.orchestra.run.smithi159:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T15:57:09.356 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:57:09.364 INFO:teuthology.orchestra.run.smithi159.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T15:57:09.696 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T15:57:09.697 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T15:57:09.697 DEBUG:teuthology.orchestra.run.smithi042:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:57:09.726 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi159.front.sepia.ceph.com 2024-09-13T15:57:09.726 DEBUG:teuthology.orchestra.run.smithi159:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T15:57:09.765 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T15:57:09.766 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:57:09.768 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T15:57:09.832 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T15:57:09.833 DEBUG:teuthology.orchestra.run.smithi042:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:57:09.853 DEBUG:teuthology.orchestra.run.smithi159:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T15:57:11.717 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T15:57:11.728 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T15:57:11.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:57:11.758 DEBUG:teuthology.orchestra.run.smithi159:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T15:57:11.786 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T15:57:11.796 DEBUG:teuthology.orchestra.run.smithi042:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:57:11.800 DEBUG:teuthology.orchestra.run.smithi159:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:57:11.837 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = core 2024-09-13T15:57:11.860 INFO:teuthology.orchestra.run.smithi159.stdout:kernel.core_pattern = core 2024-09-13T15:57:11.884 DEBUG:teuthology.orchestra.run.smithi042:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:57:11.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:57:11.932 DEBUG:teuthology.orchestra.run.smithi159:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T15:57:11.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:57:11.948 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T15:57:11.961 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T15:57:11.962 DEBUG:teuthology.misc:Transferring archived files from smithi042:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi042 2024-09-13T15:57:11.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:57:12.047 DEBUG:teuthology.misc:Transferring archived files from smithi159:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/akupczyk-2024-09-13_10:00:18-upgrade-wip-aclamk-cbt-combined-distro-default-smithi/7903557/remote/smithi159 2024-09-13T15:57:12.047 DEBUG:teuthology.orchestra.run.smithi159:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T15:57:12.094 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T15:57:12.094 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:57:12.096 DEBUG:teuthology.orchestra.run.smithi159:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T15:57:12.140 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T15:57:12.152 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T15:57:12.152 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T15:57:12.162 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T15:57:12.162 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:57:12.165 DEBUG:teuthology.orchestra.run.smithi159:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T15:57:12.185 INFO:teuthology.orchestra.run.smithi042.stdout: 266055 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:57 /home/ubuntu/cephtest 2024-09-13T15:57:12.185 INFO:teuthology.orchestra.run.smithi042.stdout: 6553601 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:09 /home/ubuntu/cephtest/mnt.0 2024-09-13T15:57:12.185 INFO:teuthology.orchestra.run.smithi042.stdout: 6553603 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 13 15:09 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T15:57:12.185 INFO:teuthology.orchestra.run.smithi042.stdout: 6698585 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:09 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T15:57:12.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T15:57:12.186 INFO:teuthology.orchestra.run.smithi042.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-13T15:57:12.187 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi042 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:57:12.193 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T15:57:12.204 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T15:57:12.217 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi042 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-13T15:57:12.222 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 6619.981970787048 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi042 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_akupczyk@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=bc62f205628c4381b2e2445577f0361d status: fail success: false 2024-09-13T15:57:12.223 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T15:57:12.248 INFO:teuthology.orchestra.run.smithi159.stdout: 265972 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 15:57 /home/ubuntu/cephtest 2024-09-13T15:57:12.336 INFO:teuthology.run:FAIL